From 34302073c2f114ce3520575b38fd5fb12780daae Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 30 Jul 2026 23:42:49 -0700 Subject: [PATCH 01/90] Add readiness checks and modify settings.gradle to comment out unused projects - Introduced `checks.py` in the `release-agent/tools` directory, implementing various readiness checks including Azure DevOps build definition access, HTTP reachability, and pipeline variable management. - Updated `settings.gradle` to comment out several project inclusions, including `AcaPlugin`, `LinuxBroker`, `java-linux-test-app`, `LinuxBrokerPackage`, and `NativeAuthSample`, to streamline the build configuration. --- .gitignore | 11 +- build.gradle | 2 +- release-agent/EXTERNAL-REFERENCES.md | 53 + release-agent/README.md | 230 +++ release-agent/config/phases.yaml | 130 ++ release-agent/config/preflight.yaml | 93 + release-agent/config/readiness.yaml | 129 ++ release-agent/config/requirements.yaml | 114 ++ release-agent/config/schedule.yaml | 20 + release-agent/orchestrator/__init__.py | 0 release-agent/orchestrator/cli.py | 53 + release-agent/orchestrator/cli_common.py | 147 ++ .../orchestrator/commands/__init__.py | 21 + .../orchestrator/commands/automation.py | 54 + .../orchestrator/commands/infra_cmd.py | 53 + .../orchestrator/commands/lockdown.py | 88 + release-agent/orchestrator/commands/logs.py | 64 + release-agent/orchestrator/commands/notice.py | 253 +++ release-agent/orchestrator/commands/notify.py | 126 ++ .../orchestrator/commands/pipeline.py | 105 ++ .../orchestrator/commands/readiness.py | 121 ++ .../orchestrator/commands/release.py | 295 ++++ release-agent/orchestrator/discovery.py | 72 + release-agent/orchestrator/engine.py | 695 ++++++++ release-agent/orchestrator/eventlog.py | 108 ++ release-agent/orchestrator/infra.py | 177 ++ release-agent/orchestrator/phase_config.py | 38 + release-agent/orchestrator/readiness.py | 201 +++ release-agent/orchestrator/registry.py | 77 + release-agent/orchestrator/render.py | 370 ++++ release-agent/orchestrator/schedule.py | 105 ++ release-agent/orchestrator/state.py | 112 ++ release-agent/phases/__init__.py | 0 release-agent/phases/agents/__init__.py | 38 + release-agent/phases/agents/preflight.py | 311 ++++ release-agent/phases/readiness_verifiers.py | 87 + release-agent/phases/stub_runner.py | 47 + release-agent/setup/bootstrap.ps1 | 86 + release-agent/skill/SKILL.md | 282 +++ .../templates/early-code-complete-notice.md | 50 + release-agent/tests/test_engine.py | 1563 +++++++++++++++++ release-agent/tools/__init__.py | 0 release-agent/tools/checks.py | 285 +++ settings.gradle | 20 +- 44 files changed, 6874 insertions(+), 12 deletions(-) create mode 100644 release-agent/EXTERNAL-REFERENCES.md create mode 100644 release-agent/README.md create mode 100644 release-agent/config/phases.yaml create mode 100644 release-agent/config/preflight.yaml create mode 100644 release-agent/config/readiness.yaml create mode 100644 release-agent/config/requirements.yaml create mode 100644 release-agent/config/schedule.yaml create mode 100644 release-agent/orchestrator/__init__.py create mode 100644 release-agent/orchestrator/cli.py create mode 100644 release-agent/orchestrator/cli_common.py create mode 100644 release-agent/orchestrator/commands/__init__.py create mode 100644 release-agent/orchestrator/commands/automation.py create mode 100644 release-agent/orchestrator/commands/infra_cmd.py create mode 100644 release-agent/orchestrator/commands/lockdown.py create mode 100644 release-agent/orchestrator/commands/logs.py create mode 100644 release-agent/orchestrator/commands/notice.py create mode 100644 release-agent/orchestrator/commands/notify.py create mode 100644 release-agent/orchestrator/commands/pipeline.py create mode 100644 release-agent/orchestrator/commands/readiness.py create mode 100644 release-agent/orchestrator/commands/release.py create mode 100644 release-agent/orchestrator/discovery.py create mode 100644 release-agent/orchestrator/engine.py create mode 100644 release-agent/orchestrator/eventlog.py create mode 100644 release-agent/orchestrator/infra.py create mode 100644 release-agent/orchestrator/phase_config.py create mode 100644 release-agent/orchestrator/readiness.py create mode 100644 release-agent/orchestrator/registry.py create mode 100644 release-agent/orchestrator/render.py create mode 100644 release-agent/orchestrator/schedule.py create mode 100644 release-agent/orchestrator/state.py create mode 100644 release-agent/phases/__init__.py create mode 100644 release-agent/phases/agents/__init__.py create mode 100644 release-agent/phases/agents/preflight.py create mode 100644 release-agent/phases/readiness_verifiers.py create mode 100644 release-agent/phases/stub_runner.py create mode 100644 release-agent/setup/bootstrap.ps1 create mode 100644 release-agent/skill/SKILL.md create mode 100644 release-agent/templates/early-code-complete-notice.md create mode 100644 release-agent/tests/test_engine.py create mode 100644 release-agent/tools/__init__.py create mode 100644 release-agent/tools/checks.py diff --git a/.gitignore b/.gitignore index 4618e843..2802f33f 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,13 @@ ehthumbs_vista.db out/ -plugins/buildsystem/bin \ No newline at end of file +plugins/buildsystem/bin + +ICM-investigation/ +# Release Orchestrator generated run-state (per-release, ephemeral) +.release-runs/ + +# Python bytecode +__pycache__/ +*.pyc + diff --git a/build.gradle b/build.gradle index a3e5b735..90a3f6b3 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ buildscript { dependencies { classpath "com.android.tools.build:gradle:${rootProject.ext.gradleVersion}" classpath "org.javassist:javassist:${rootProject.ext.javaAssistVersion}" - classpath "com.microsoft.intune.mam:android-build-plugin:${rootProject.ext.intuneAppSdkVersion}" + // classpath "com.microsoft.intune.mam:android-build-plugin:${rootProject.ext.intuneAppSdkVersion}" classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${rootProject.ext.kotlinVersion}" // classpath "net.serenity-bdd:serenity-gradle-plugin:1.9.6" // classpath 'com.google.gms:google-services:3.2.1' diff --git a/release-agent/EXTERNAL-REFERENCES.md b/release-agent/EXTERNAL-REFERENCES.md new file mode 100644 index 00000000..f4b5b59a --- /dev/null +++ b/release-agent/EXTERNAL-REFERENCES.md @@ -0,0 +1,53 @@ +# External References + +Everything the Release Orchestrator depends on that lives **outside this codebase**. +If any of these change (URL moved, DL renamed, pipeline re-IDed, template edited, +access revoked), the orchestrator can silently break — so they're catalogued here. +Review this list when something stops working or when onboarding a new release owner. + +Legend for **Access**: `anon` = no auth · `az` = Azure CLI signed-in user · +`AAD-SSO` = browser Microsoft sign-in · `MCP` = via an MCP server · `Google` = Google account (not automatable in Scout). + +## Systems of record (read/write) + +| Ref | What | Used by | Access | Notes | +|---|---|---|---|---| +| ADO pipeline **3038** | "Code Complete Calendar Checker" — CCD source of record | CCD seed, `set-ccd`, `skip-release`, Phase-0 `cron` (verify scheduled) | az | org identitydivision / project Engineering. Real writes gated by --confirm. `cron` step verifies a recent `schedule`-reason run. | +| ADO build def **2828** | Auth Client Android build (org identitydivision / project Engineering) | readiness `build_access` | az | access check only | +| ADO build def **397224** | Android Build Release (org msazure / project One) | readiness `build_access` | az | access check only | +| ADO wiki **IdentityWiki.wiki** page **59148** | "Monthly Releases Payloads History" (parent) | Phase-0 `wiki` agent | az (`az devops wiki`) | child page ` Release`; dup-safe numbering | +| **ICM team 78848** | "Auth Client Android Shield" on-call roster | readiness `oncall_now` | MCP (ICM) | primary = index 0 of currentOnCallContacts | +| **ADX cluster** idsharedeus2.eastus2 / db d496be22d62a46b0a3cf67ea2e736fd8 | release telemetry | readiness `adx_access` | MCP (Kusto) | `print 1` access probe | + +## External web pages (scraped / linked) + +| Ref | URL | Used by | Access | Notes | +|---|---|---|---|---| +| CCOA No-Fly Zones | https://prod.change-manager.msidentity.com/ccoa-periods | Phase-0 `lockdown` | AAD-SSO | scraped via browser; only Production-env periods block | +| **Component Governance alerts** (governed repo **104410** = AD-MFA-phonefactor-phoneApp-android, branch `working`) | https://msazure.governance.visualstudio.com/{One projId}/_apis/ComponentGovernance/GovernedRepositories/104410/Branches/working/Alerts | Phase-0 `cg` | az (`az rest`) | read-only report; active alerts by severity. projId=b32aa71e-…, resource=499b84ac-… | +| **`release` variable group 40** | https://identitydivision.visualstudio.com/Engineering/_library?...variableGroupId=40&path=release | Phase-0 `flight_reminder` (link only) | az/web | feature owners update local flights here — release engineer does NOT | +| Flight pre-mortem example doc | https://microsoft-my.sharepoint-df.com/:w:/p/rapong/cQpEZp0cXp1sQYo4A4M3PQWCEgUCDj364FJa-rq-msg59WlBsw | Phase-0 `flight_reminder` (link only) | AAD-SSO | example shared with feature owners | +| Localization instructions | https://eng.ms/docs/.../combined-release-checklist/localization | Phase-0 `flight_reminder` (link only) | AAD-SSO | confirmed valid 2026-07-29 | +| **Teams chat: "Android Core Team"** | thread `19:976a859f167f44e59c4ceca8b1d23581@thread.v2` | Phase-0 `flight_reminder` target | MCP (WorkIQ) | LIVE target; dry-run posts to the owner's own chat | +| **EcsFlight.kt** (Auth App ECS flights) | https://msazure.visualstudio.com/One/_git/AD-MFA-phonefactor-phoneApp-android?path=/.../ecs/entities/EcsFlight.kt&version=GBworking | Phase-0 `flight_reminder` bullet 4 (link only) | az/web | reviewers check its history since last code complete | +| Early code-complete notice template | https://eng.ms/docs/.../combined-release-checklist/early-code-complete-notice-email-template | Phase-0 `notice` | AAD-SSO | copied locally to `templates/early-code-complete-notice.md` — **re-sync if upstream edits** | +| Hotfix cherry-pick guide | https://eng.ms/docs/.../release/cherry-pick-to-hotfix-guidelines | link inside notice email body | AAD-SSO | referenced, not fetched | +| common-for-android changelog | https://raw.githubusercontent.com/AzureAD/microsoft-authentication-library-common-for-android/dev/changelog.txt | Phase-0 `breaking` | anon | breaking = `[MAJOR]` in `vNext` | +| Play Console vitals | (Google Play Console) | Phase-0 `vitals` (#8) | Google | **NOT automatable in Scout** (Google auth wall) | + +## Outbound email recipients (LIVE runs only) + +> In **dry-run**, every outbound email goes to the **release owner** instead of these. +> Real recipients are used only on a `--live` release. + +| Step | To | Notes | +|---|---|---| +| Phase-0 `notice` (early code-complete) | androididentity@microsoft.com ("Azure Identity Android SDK"), jialh@microsoft.com | provided by release owner 2026-07-29 | + +## Tooling / infra (provisioned by bootstrap) + +| Ref | What | Notes | +|---|---|---| +| Agency CLI | provides the **ICM** and **Kusto** MCP servers | `agency mcp icm` / `agency mcp kusto`; auto-registered into `~/.scout/m-mcp-servers.json` by `cli infra` | +| Azure CLI (`az`) + `azure-devops` extension | pipeline + wiki reads/writes | signed-in user is the release owner | +| Scout | host for the skill + automations | `~/.scout`; bootstrap checks presence | diff --git a/release-agent/README.md b/release-agent/README.md new file mode 100644 index 00000000..be2932ad --- /dev/null +++ b/release-agent/README.md @@ -0,0 +1,230 @@ +# Release Orchestrator (`/release-agent`) + +The **conductor backbone** for the Android monthly release (ADO items **X4** + **X5**). +It drives the whole release as a state-aware smart checklist: it knows the phases, +runs each step's agent, and **holds at gates** for the release engineer to decide. + +> **Build status.** **Phase 0 (pre-flight) has real, tested agents** (early-notice, +> flight/string reminders, BREAKING-OneAuth detection, CG alerts, cron verify, wiki +> payload); the later phases are still **stubs** (mock actions) and get filled in one +> at a time (see the Release-Stabilization roadmap). Building on this shared backbone — +> not 50 one-off scripts — is what makes the "agent carries the knowledge" model real. + +## Architecture (thin skill over a deterministic engine) + +``` + you ──/release-agent──▶ SKILL (conversation layer) ──shell──▶ ENGINE (Python, deterministic) + presents gate briefs, state machine + dispatch + run-state + relays your approve/deny the BRAIN — fully unit-tested +``` + +- **Engine = the brain.** Decides what's next, runs stubbed agents, holds at gates, persists run-state. No LLM — unit-tested and dry-run replayable. +- **Skill = the mouth & ears.** Presents the gate, collects your decision, relays it. Never decides the flow. + +## Layout + +``` +release-agent/ COMMITTED (distributed with android-complete) +├─ config/ +│ ├─ phases.yaml the state machine (phases → steps → gates + CCD anchors), as data +│ ├─ preflight.yaml Phase-0 config (config/.yaml convention; loaded by phase id) +│ ├─ readiness.yaml the entry-gate checklist, as data +│ ├─ schedule.yaml where CCD comes from (pipeline 3038 coords), as data +│ └─ requirements.yaml external dependencies (CLIs, extensions, MCP servers) — single source of truth +├─ orchestrator/ three layers: logic → data → presentation +│ ├─ engine.py the conductor: state machine + dispatch + gates + time-anchoring + status_report +│ ├─ readiness.py ReadinessGate: entry-gate logic (verify/sign/decline) → structured data +│ ├─ schedule.py CCD math (2nd Wednesday, override, phase anchors) — pure, no IO +│ ├─ phase_config.py per-phase config loader (config/.yaml, by phase id) +│ ├─ infra.py infra preflight: check CLIs + register/verify MCP servers into Scout config +│ ├─ render.py presentation only: structured data → text/markdown (swap for other UIs) +│ ├─ state.py Release State Record / run-state (X5) +│ ├─ discovery.py find releases (none / one / many) +│ ├─ registry.py automation registry (track provisioned automations for teardown) +│ ├─ eventlog.py per-release interaction + event log +│ ├─ cli.py thin entry point: builds the parser from commands/, dispatches +│ ├─ cli_common.py shared CLI plumbing (load state, emit, event log, advance block) +│ └─ commands/ one module per command domain (self-registering subparsers) +│ ├─ release.py lifecycle + overrides: init/list/status/next/approve/deny/done/skip/reopen/halt/resume/activate +│ ├─ readiness.py entry gate: checklist/verify/sign/decline +│ ├─ pipeline.py real pipeline writes (gated): set-ccd / skip-release +│ ├─ notify.py daily phase digest (tick advances + reports; notify = read-only) + set-owner +│ ├─ lockdown.py CCOA overlap check: check-lockdown +│ ├─ notice.py Phase-0 scout steps: prepare-notice / prepare-flight-reminder / record-step +│ ├─ logs.py event log: log / journal +│ ├─ automation.py automation registry command +│ └─ infra_cmd.py infra preflight command +├─ phases/ +│ ├─ stub_runner.py mock phase agents (replaced one at a time) +│ ├─ readiness_verifiers.py auto verifiers for the entry gate (pass/fail) +│ └─ agents/ real phase agents — one module per phase, merged into one registry +│ ├─ __init__.py aggregator: merges every phase's REGISTRY (dup-id guarded) +│ └─ preflight.py Phase-0 agents (breaking · wiki · cg · cron) +├─ tools/checks.py real IO (az / http), isolated +├─ skill/SKILL.md the /release-agent Scout skill +├─ setup/bootstrap.ps1 one-time setup (infra preflight, installs skill) +└─ tests/test_engine.py unit + full dry-run-replay tests + +.release-runs// GENERATED, gitignored (per-release working state) +├─ release-state.json the per-release metadata + run-state (owner, CCD, steps, gates, …) +└─ events.jsonl the per-release event/interaction log +.release-runs/_automations.json GENERATED, gitignored — registry of provisioned Scout automations +``` + +**Two homes for data (by lifetime):** +- **Release metadata + run-state** → `.release-runs//release-state.json` (per-release; the `ReleaseState` record). Holds `owner_email`/`owner_name` (the release owner, resolved from the signed-in `az` user at `init`; reminders email this person), `ccd`/`ccd_source`/`ccd_conflict`, `dry_run`, step completion, gate decisions, `last_notified`, etc. Add release-scoped fields here. +- **Tool config** → `release-agent/config/*.yaml` (not release-specific; committed): `phases.yaml`, `readiness.yaml`, `schedule.yaml`, `requirements.yaml`. + +## Architecture — three layers (so it adapts to other interfaces) + +1. **Logic** (`engine.py`, `readiness.py`, `schedule.py`, `state.py`) — pure, deterministic, returns **structured data**. No formatting, no IO. +2. **Presentation** (`render.py`) — pure functions: structured data → text/markdown. A different interface (web UI, TUI) swaps this layer and reuses everything else. +3. **Interface** (`cli.py` + `cli_common.py` + `commands/` + `skill/SKILL.md`) — the CLI is a thin assembler: `cli.py` builds the parser from the self-registering modules in `commands/` (one per domain), and shared plumbing lives in `cli_common.py`. Adding a command is a localized change to one module. + +IO lives in `tools/` and `phases/` (pluggable). Config is data in `config/`. + + +## Run-state: two kinds (the X5 idea) + +- **Derived** — recomputed from systems of record (ADO/Git/Play Console/ADX). Never stored ⇒ never stale. *(reconcilers are stubbed for now.)* +- **Persisted** — decisions/intent, step completion, pending human actions. Stored in `release-state.json`. + +The conductor is **stateless**: on each invocation it loads the record, (later) reconciles against live systems, decides, acts, writes back. That's what lets a release resume across days/sessions. + +## Quick start + +```powershell +# one-time +pwsh ./setup/bootstrap.ps1 +``` + +`bootstrap.ps1` runs an **infrastructure preflight** first (`python -m orchestrator.cli infra`), +driven by **`config/requirements.yaml`** (the single source of truth for external +dependencies). It checks each CLI/host dependency and prints an `install:` hint for +anything missing, then **registers any required MCP servers into Scout's config** +(backing the file up first) and tells you to **restart Scout** so they load. Keep +`requirements.yaml` up to date whenever a new dependency (CLI, package, or MCP +server) is introduced. + +You can run the preflight any time on its own: + +```powershell +python -m orchestrator.cli infra # check + auto-register MCP servers (restart Scout after) +python -m orchestrator.cli infra --no-register # report only +``` + +```powershell +# drive a release (dry-run by default) +cd release-agent +python -m orchestrator.cli init --release 2026-07 +python -m orchestrator.cli next --release 2026-07 # runs until the first gate +python -m orchestrator.cli approve --release 2026-07 --comment "flags reviewed" +python -m orchestrator.cli status --release 2026-07 +``` + +Or in Scout: **`/release-agent`**. + +## Time anchoring — phases open relative to the Code Complete Date (CCD) + +Phases don't fire on demand; they're anchored to the **CCD**. **The CCD is +canonically the 2nd Wednesday of the month.** The orchestrator still reads ADO +pipeline **3038 "Code Complete Calendar Checker"**, but it does **not** silently +adopt the pipeline's `overrideCodeCompleteDate`: if that override is a *different* +in-month date, the tool flags a **conflict** (`ccd_conflict`) and asks the user +which date is real — the default or the pipeline's. `init` computes the default +and reports any conflict. + +- **Phase 0 opens at `CCD-7`** (declared as `anchor: "CCD-7"` on the phase in + `phases.yaml`). Before then the release is **`scheduled`** — the engine runs + nothing and status shows *"opens `` (in N days)"*. Other phases are + dependency-driven for now; add an `anchor:` to any phase to time-gate it too. +- **Simulated clock:** every read/advance command takes `--as-of YYYY-MM-DD` so a + dry-run can jump to CCD-7 and prove a phase opens on schedule. Real runs use today. +- **Resolving a conflict / changing the CCD.** `set-ccd` and `skip-release` + **write back** to pipeline 3038 (override / `skipRelease`) — real production + changes, so they're gated: preview first, then re-run with `--confirm` (a + `--reason` is always required and audited). Pick the default → `set-ccd --default` + clears the pipeline override so they match; pick the pipeline date → `set-ccd + --date `. `status` re-reads the pipeline and re-flags any new conflict. + +```powershell +python -m orchestrator.cli set-ccd --release 2026-07 --date 2026-07-15 --reason "more bake time" # preview +python -m orchestrator.cli set-ccd --release 2026-07 --date 2026-07-15 --reason "more bake time" --confirm +python -m orchestrator.cli status --release 2026-07 --as-of 2026-07-01 # jump the clock in dry-run +python -m orchestrator.cli done --release 2026-07 --note "China upload complete" # clear a reminder hold +``` + +## Push reminders — daily phase digest (reaching you when Scout is closed) + +Everything the engine surfaces is **pull** — you see it when you open Scout. The +**push** layer is a **daily phase status digest** emailed to the release owner: + +- **Setup is interactive → no push.** Readiness + establishing the CCD happen in + Scout, so they're never emailed (unsigned / blocked / halted = silent). +- **First push = a phase opening** (Phase 0 at CCD‑7). Nothing before it. +- **Daily while a phase is open with outstanding work** — once/day, progress + + what still needs you, until the phase's actions are done; then the next phase's + digest takes over when it opens (each phase notifies on open). + +```powershell +python -m orchestrator.cli tick --json # advance to today + {message,subject,owner_email,...} +python -m orchestrator.cli tick --as-of 2026-08-06 # simulate a date (debug) +python -m orchestrator.cli notify --json # read-only: report WITHOUT advancing (manual check) +``` + +A **Scout automation** runs **`tick --json` hourly** and, when `message` is non‑empty, +emails it to `owner_email` (subject from the JSON). `tick` both **advances** the release +to the current date and reports; running hourly means a tick missed while the machine +was off is picked up by the next one, and a once-per-calendar-day guard +(`last_notified_date`) keeps it to one advance-effect and one email per day. `notify` is +the **read-only** variant (report without advancing); `--as-of`/`--force` are debug overrides. + +**Automation registry.** Every automation the orchestrator provisions is recorded in +`.release-runs/_automations.json` (via `cli automation register`) so it can be torn +down cleanly. Automations are **per-release** by default (`--release `), created +at start and removed at that release's close (`automation list --release ` → +delete each → `automation deregister`). Push reminders are per-release too. A +`--shared` scope exists for the rare automation meant to outlive every release. + +## Two kinds of human step + +- **Gate** (`gate: true`) — a *decision*: the conductor holds and you `approve`/`deny`. +- **Reminder** (`owner: human`, no gate) — a *to-do*: the conductor holds + ("ACTION NEEDED"), you go do it, then `done` it. Not a decision — just done / not-yet. + +## Event log (for analysis & improvement) + +Every action is recorded to an append-only JSONL event log so we can improve the +process across engineers and months. The highest-value signal is the **decision +driver** — the reason attached to each gate approve/deny/decline. + +- Per-release trace: `.release-runs//events.jsonl` (one log per release; there is no machine-wide aggregate). + +```powershell +python -m orchestrator.cli log --release 2026-07 # this release's trace +python -m orchestrator.cli log --release 2026-07 --analyze # rolled-up summary +``` + +Events captured include: `release_started`, `readiness_verified/signed/declined`, +`step_ran`, `gate_hold` + `gate_approved`/`gate_denied` (with `driver`), +`reminder_hold`/`reminder_done`, `scheduled_hold`, `ccd_changed`, +`release_skip_set`/`release_skip_cleared`, `step_skipped`/`step_reopened`, +`release_halted`/`release_resumed`, `release_complete`, plus interaction events +(what Scout showed / what the user chose). Logging never breaks the flow (best-effort). + +> The log lives under the gitignored `.release-runs/`, so it's per-machine. Shipping +> logs to a shared store (Kusto/ADO/wiki) for cross-engineer analysis is a future step. + +## Tests + +```powershell +cd release-agent +python tests/test_engine.py # unit + full dry-run replay + readiness + eventlog +``` + +## Design constraints honored (from §7.1 of the stabilization plan) +1. Dry-run/replay is the primary test method (never test on live monthly releases). +2. Run-state schema defined once, upfront (X5), shared by all agents. +3. Sequence by risk/value — agents are independent plug-ins on the backbone. +4. Manual overrides are first-class (approve/deny gates; activate conditional phases). +5. Conductor is stateless; minimize persisted state, derive the rest. diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml new file mode 100644 index 00000000..7a551a30 --- /dev/null +++ b/release-agent/config/phases.yaml @@ -0,0 +1,130 @@ +# Release Orchestrator — state machine definition (data, not code). +# The conductor reads this to know phases -> steps -> gates -> transitions. +# Phase 0 (preflight) has REAL agents (see phases/agents/preflight.py); later +# phases are still `agent: stub` and get a real agent id as each is implemented. +# +# Field reference: +# phases[].id/name : phase identifier + label +# phases[].checklist_phase : maps to the human checklist Phase number +# phases[].execution : sequential (default) | parallel (independent steps) +# steps[].id/name : step identifier + label +# steps[].agent : which phase-agent handles it ('stub' until built) +# steps[].gate : if true, conductor HOLDS at this step for human approval +# steps[].owner : agent | human (who acts; gates are always human-decided) +# steps[].source : scout -> the skill runs it via MCP + record-step +# steps[].attest : if true, a human confirms it (owner: human) +# steps[].depends_on : step ids in this phase that must finish first +# steps[].maps_to : the Release-Stabilization action-item ID(s) this step uses + +version: 1 + +phases: + - id: preflight + name: "Pre-flight & Code Complete" + checklist_phase: 0 + anchor: "CCD-7" # opens 7 days before the Code Complete Date (start window) + execution: parallel # steps are independent; a hold on one doesn't block the others + steps: + - { id: notice, name: "Send early release notice", agent: stub, owner: agent, source: scout, maps_to: [S0] } + - { id: flight_reminder, name: "Send feature-owner reminders (flights · strings · flag-freeze)", agent: stub, owner: agent, source: scout, maps_to: [S3, S4, S5] } + - { id: confirm_reminders, name: "Confirm feature owners completed flight / string / flag-freeze work", agent: stub, owner: human, attest: true, depends_on: [flight_reminder], maps_to: [S3, S4, S5, S6] } + - { id: lockdown, name: "Detect lockdown/holiday overlap", agent: stub, owner: agent, source: scout, maps_to: [S1] } + - { id: breaking, name: "Detect BREAKING-OneAuth + draft comms", agent: breaking_detect, owner: agent, maps_to: [S2] } + - { id: cg, name: "Report critical CG alerts", agent: cg_alerts, owner: agent, maps_to: [S8] } + - { id: vitals, name: "Confirm Play Console vitals & policy status reviewed", agent: stub, owner: human, attest: true, maps_to: [S9] } + - { id: cron, name: "Verify Calendar Checker scheduled", agent: cron_check, owner: agent, maps_to: [S10] } + - { id: wiki, name: "Create release payload wiki subpage", agent: wiki_payload, owner: agent, maps_to: [S0] } + + - id: ccd + name: "Code Complete Day" + checklist_phase: 1 + steps: + - { id: final_reminder, name: "CCD final code-complete reminder", agent: stub, owner: agent, maps_to: [P1-1] } + - { id: localization, name: "Trigger localization pipeline (PR)", agent: stub, owner: agent, maps_to: [P1-2] } + - { id: precheck_prs, name: "Pre-check open required PRs", agent: stub, owner: agent, maps_to: [P1-3a] } + - { id: branch_cut, name: "Cut the release branch", agent: stub, owner: human, gate: true, maps_to: [P1-3b] } + - { id: verify_trigger, name: "Verify orchestrator fired overnight", agent: stub, owner: agent, maps_to: [P1-4] } + + - id: build_verify + name: "Build & Lib Verification" + checklist_phase: 2 + steps: + - { id: stages_ok, name: "Verify pipeline completed expected stages", agent: stub, owner: agent, maps_to: [B1] } + - { id: retain, name: "Retain the build", agent: stub, owner: agent, maps_to: [B1r] } + - { id: health, name: "Assess build-completeness + product health", agent: stub, owner: agent, maps_to: [B2] } + - { id: ui_auto, name: "Assess UI automation results", agent: stub, owner: agent, maps_to: [B3] } + - { id: payload, name: "Write built versions into payload wiki", agent: stub, owner: agent, maps_to: [B6] } + - { id: mrwp_rc, name: "Point MRWP at cut RC Auth App branch", agent: stub, owner: agent, maps_to: [B4] } + - { id: go_test, name: "Proceed to bug bash", agent: stub, owner: human, gate: true } + + - id: bug_bash + name: "Test / Bug Bash" + checklist_phase: 3 + steps: + - { id: clone_plans, name: "Clone/rename test plans", agent: stub, owner: agent, maps_to: [T3] } + - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", agent: stub, owner: agent, maps_to: [T2] } + - { id: ui_failures, name: "Surface Phase 2 UI failure list", agent: stub, owner: human, maps_to: [T4] } + - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", agent: stub, owner: agent, maps_to: [T5] } + - { id: bash_done, name: "Bug bash complete + signed off", agent: stub, owner: human, gate: true } + + - id: finalize + name: "Finalize & Publish" + checklist_phase: 4 + steps: + - { id: gate_watch, name: "Watch orchestrator gates (1-click approve)", agent: stub, owner: human, gate: true, maps_to: [F1] } + - { id: integ_prs, name: "Auto-create integration PRs", agent: stub, owner: agent, maps_to: [F2] } + - { id: verify_pub, name: "Verify Maven Central + GitHub publication", agent: stub, owner: agent, maps_to: [F3] } + - { id: final_comms, name: "Send final broker email + Teams", agent: stub, owner: agent, maps_to: [F4] } + - { id: nonrc_pin, name: "Final non-RC pin PR + kick + retain", agent: stub, owner: agent, maps_to: [F5] } + - { id: tag, name: "Tag the release commit", agent: stub, owner: agent, maps_to: [F6] } + - { id: backmerge, name: "Back-merge to working", agent: stub, owner: agent, maps_to: [F7] } + + - id: rollout_start + name: "Rollout Start" + checklist_phase: 5 + steps: + - { id: notice, name: "Send initial Auth App release notice", agent: stub, owner: agent, maps_to: [R1] } + - { id: artifact, name: "Verify final non-RC artifact is used", agent: stub, owner: agent, maps_to: [R2] } + - { id: signoff_start, name: "Start Release Sign Off (1-click)", agent: stub, owner: human, gate: true, maps_to: [R3] } + + - id: monitor + name: "Monitoring & Ring Advancement" + checklist_phase: 6 + steps: + - { id: health_report, name: "Crash-% health report (halt/hotfix/proceed)", agent: stub, owner: agent, maps_to: [M1] } + - { id: adoption, name: "Adoption % vs ring threshold", agent: stub, owner: agent, maps_to: [M2] } + - { id: guards, name: "Check advancement guard rules", agent: stub, owner: agent, maps_to: [M4] } + - { id: advance, name: "Ring advance / halt decision", agent: stub, owner: human, gate: true, maps_to: [M3] } + - { id: progress_email, name: "Send progression email on advance", agent: stub, owner: agent, maps_to: [M5] } + - { id: dashboard, name: "Update ADX dashboard params", agent: stub, owner: agent, maps_to: [M6] } + - { id: crash_annot, name: "Flag new vs existing crashes", agent: stub, owner: human, maps_to: [M7] } + - { id: hotfix_eval, name: "Surface hotfix-trigger criteria", agent: stub, owner: human, maps_to: [M8] } + + - id: partner + name: "Partner Stores" + checklist_phase: 7 + steps: + - { id: china, name: "China publish flow", agent: stub, owner: human, maps_to: [PT1] } + - { id: samsung, name: "Samsung portal upload", agent: stub, owner: human, maps_to: [PT1s] } + - { id: ngms, name: "Verify nGMS upload", agent: stub, owner: agent, maps_to: [PT2] } + - { id: teams_dev, name: "Teams Devices announcement", agent: stub, owner: agent, maps_to: [PT3] } + - { id: approvals, name: "Track partner approvals + final notice", agent: stub, owner: agent, maps_to: [PT4] } + - { id: review_times, name: "Maintain review-times table", agent: stub, owner: agent, maps_to: [PT5] } + + - id: hotfix + name: "Hotfix (conditional)" + checklist_phase: 8 + conditional: true + steps: + - { id: cherry, name: "Cherry-pick PRs", agent: stub, owner: agent, maps_to: [H1] } + - { id: rebuild, name: "Rebuild + retain hotfix build", agent: stub, owner: agent, maps_to: [H2] } + - { id: smoke, name: "P0 smoke subset", agent: stub, owner: human, gate: true, maps_to: [H3] } + - { id: expedite, name: "Expedited rollout", agent: stub, owner: agent, maps_to: [H4] } + - { id: alpha_orgs, name: "Track alpha-mitigation orgs to remove", agent: stub, owner: agent, maps_to: [H5] } + + - id: close + name: "Release Close" + checklist_phase: 9 + steps: + - { id: closeout, name: "Close-out checklist (approvals/comms/bugs)", agent: stub, owner: agent, maps_to: [C1] } + - { id: done, name: "Release complete", agent: stub, owner: human, gate: true } diff --git a/release-agent/config/preflight.yaml b/release-agent/config/preflight.yaml new file mode 100644 index 00000000..3f1ba65f --- /dev/null +++ b/release-agent/config/preflight.yaml @@ -0,0 +1,93 @@ +# Config for the REAL Phase-0 pre-flight agents (deterministic; az CLI / HTTP). +# +# Unlike the readiness scout-assisted checks (ICM/Kusto, which have no CLI and are +# run by the skill via MCP), these steps talk to systems that DO have a CLI/HTTP +# surface, so they are plain Python agents — testable and LLM-free. In a dry-run +# release they SIMULATE (no network, no writes); only a real release acts. + +version: 1 + +# Step `cg` (maps_to S8): "Report critical CG alerts". +# Component Governance alerts for the governed repo, read (read-only) from the CG +# governance host via `az rest`. Reports ACTIVE alerts grouped by severity and +# surfaces High/Critical. Report-only — never blocks the release. +cg: + resource: "499b84ac-1321-427f-aa17-267ca6975798" # Azure DevOps resource id (for `az rest`) + governance_host: "https://msazure.governance.visualstudio.com" + project_id: "b32aa71e-8ed2-41b2-9d77-5bc261222004" # msazure/One + governed_repo_id: 104410 # AD-MFA-phonefactor-phoneApp-android + branch: "working" + high_severities: ["critical", "high"] # surfaced/flagged as high-priority + +# Step `cron` (maps_to S10): "Verify Calendar Checker scheduled". +# Pipeline 3038's cron lives in YAML (not exposed in definition triggers), but its +# build history proves the schedule is FIRING: a recent `schedule`-reason run means +# the Calendar Checker is live. Blocks if there's no recent scheduled run (stale). +cron: + pipeline_id: 3038 + org: "https://identitydivision.visualstudio.com" + project: "Engineering" + name: "Code Complete Calendar Checker" + max_staleness_days: 2 # a daily cron should never be older than this + + +# Step `breaking` (maps_to S2): "Detect BREAKING-OneAuth + draft comms". +# common-for-android records breaking changes as [MAJOR] entries in changelog.txt. +# The unreleased "vNext" section holds the changes shipping in THIS release, so we +# scan only that section. Any [MAJOR] there is a breaking change OneAuth must hear +# about — the agent lists them and drafts the comms; a human sends it. +breaking: + changelog_url: "https://raw.githubusercontent.com/AzureAD/microsoft-authentication-library-common-for-android/dev/changelog.txt" + section: "vNext" # scan the unreleased section only + breaking_tag: "[MAJOR]" # breaking-change marker convention in this changelog + +# Step `wiki` (maps_to S0): "Create release payload wiki subpage". +# Creates the per-release payload page under the standing history parent page. +# Phase 2 (step B6) later writes the built versions into this same page. +wiki: + org: "https://identitydivision.visualstudio.com" + project: "IdentityWiki" + wiki: "IdentityWiki.wiki" + parent_path: "/IdentityWiki/Services/Microsoft Authenticator/Release/Android/Monthly Releases Payloads History" + # Child page name = " Release" (e.g. "August 2026 Release"). + # If that page already exists, the agent NOTIFIES and creates the next free + # numbered page instead ("August 2026 2 Release", "August 2026 3 Release", ...). + +# Step `lockdown` (maps_to S1): "Detect lockdown/holiday overlap". +# The CCOA "No-Fly Zones" source is an AAD-gated web app, so it can't be read by +# deterministic Python — the SKILL scrapes it via the authenticated browser and +# passes the periods to `check-lockdown`, which decides overlap DETERMINISTICALLY. +# Only periods whose Environment is Production BLOCK the release; Banner-only +# advisories are ignored. Overlap is checked against the release window +# CCD-7 .. CCD+14. On overlap the step holds for the owner (who runs set-ccd). +lockdown: + url: "https://prod.change-manager.msidentity.com/ccoa-periods" + blocking_environment: "Production" # only Production-env CCOA periods block + window_start_anchor: "CCD-7" + window_end_anchor: "CCD+14" + +# Step `notice` (maps_to S0): "Send early release notice". +# Fills the local template deterministically and sends it. RECIPIENTS: in a +# dry-run every outbound email goes to the release owner (safe rehearsal); a live +# release uses the real `recipients` list below. See EXTERNAL-REFERENCES.md. +notice: + template: "templates/early-code-complete-notice.md" + variant: "initial" # initial (CCD-7) | update (CCD-day) + recipients: # LIVE only; dry-run redirects to owner + - "androididentity@microsoft.com" # "Azure Identity Android SDK" + - "jialh@microsoft.com" + +# Step `flight_reminder` (maps_to S3/S4/S5): "Feature-owner flight & string reminders". +# Three checklist reminders combined into ONE Teams message to the Android Core Team +# group chat. Scout-assisted (Teams send needs WorkIQ). DRY-RUN → the message goes to +# the release owner's own Teams chat; LIVE → the group chat below. See EXTERNAL-REFERENCES.md. +flight_reminder: + live_chat_id: "19:976a859f167f44e59c4ceca8b1d23581@thread.v2" # "Android Core Team" + live_chat_name: "Android Core Team" + links: + variable_group: "https://identitydivision.visualstudio.com/Engineering/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=40&path=release" + premortem_example: "https://microsoft-my.sharepoint-df.com/:w:/p/rapong/cQpEZp0cXp1sQYo4A4M3PQWCEgUCDj364FJa-rq-msg59WlBsw" + localization: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/combined-release-checklist/localization" + ecs_flight_history: "https://msazure.visualstudio.com/One/_git/AD-MFA-phonefactor-phoneApp-android?path=/PhoneFactor/ExperimentationLibrary/src/main/java/com/microsoft/authenticator/experimentation/ecs/entities/EcsFlight.kt&version=GBworking" + + diff --git a/release-agent/config/readiness.yaml b/release-agent/config/readiness.yaml new file mode 100644 index 00000000..bd9edd3c --- /dev/null +++ b/release-agent/config/readiness.yaml @@ -0,0 +1,129 @@ +# Release readiness checklist — the ENTRY GATE. +# The engine blocks (status = readiness_gate) until EVERY item is satisfied. +# All items are equally required — there is no per-item priority. +# +# The ONLY distinction is WHO resolves an item (the `verify` field): +# auto -> Scout resolves it (verifies programmatically). Result is pass or fail; +# if it can't be fully proven, it is NOT auto. Two execution sources: +# * (default) a Python verifier in phases/readiness_verifiers.py (uses az/http) +# * source: scout -> the SKILL runs the check via its MCP tools (e.g. ICM) +# and records the result with `record-check`; the Python engine skips it. +# attest -> The engineer resolves it (confirms). Used for anything Scout cannot +# fully prove (physical devices, portal sign-in). +# +# If any item is unsatisfied the gate stays closed. If the engineer cannot satisfy +# an attest item, they resolve it or hand the release to someone who can. +# +# Edit this file to change the entry checklist — it is data, not code. + +title: "Release readiness — entry gate" +instructions: > + Clear every item before Phase 0 can start. Scout verifies the auto items; you + attest the rest. Every item is required — the release cannot start until all are + satisfied. +blocked_message: > + This item is required and is not satisfied, so the release cannot start. Resolve + it — or, if you cannot, hand the release to another engineer who can (notify your + manager / the release team). The new owner runs /release-agent and completes the + checklist themselves. + +items: + # ---- AUTO: fully verified by Scout (pass/fail, no half-measures) ---- + # `label` = short name for the table; `detail` = PLAIN cell text (no inline links — + # markdown links don't render inside Scout table cells). Links go in `links` + # (list of {name,url}) and are rendered as a clickable reference list BELOW the table. + # For build_access the detail + links are generated from the live check results. + - id: build_access + label: "Build definitions accessible" + text: "Release build definitions are accessible" + verify: auto + verifier: build_defs + checks: + - type: ado_build_def + name: "Auth Client Android (Engineering #2828)" + org: "https://identitydivision.visualstudio.com/" + project: "Engineering" + id: 2828 + url: "https://identitydivision.visualstudio.com/Engineering/_build?definitionId=2828" + - type: ado_build_def + name: "Android Build Release (One #397224)" + org: "https://msazure.visualstudio.com/" + project: "One" + id: 397224 + url: "https://msazure.visualstudio.com/One/_build?definitionId=397224" + + # Everything the tool needs to run UNATTENDED (machine on, Scout not focused): + # the MCP servers registered, and the permissions set so scheduled work (the + # daily digest, Teams reminders, browser checks) runs without a prompt. + - id: mcp_servers + label: "MCP servers registered" + text: "The MCP servers the release needs (ICM on-call, Kusto/ADX telemetry) are registered in Scout" + detail: "Scout checks the ICM + Kusto/ADX MCP servers are registered in your Scout config (needed for the on-call and telemetry checks)." + verify: auto + verifier: mcp_servers + + - id: silent_perms + label: "Silent-run permissions" + text: "Permissions allow fully unattended runs — shell, WorkIQ (email + Teams), and the browser are auto-approved so scheduled work never stalls on a prompt" + detail: "Scout checks shell, WorkIQ (email + Teams), and the browser are all auto-approved so the daily automation runs silently when Scout isn't focused." + verify: auto + source: scout # only the skill can read Scout's own settings (m_get_settings) + verifier: silent_perms + # The servers that must be auto-approved for a fully-silent run (kept as data). + required_servers: [shell, workiq, playwright] + # OPT-OUT (soft): enabling silent runs needs the user to turn on the Scout master + # toggle "Allow AI to request permission changes" (allowModelPermissionsChange) — + # which ONLY the user can flip in the UI. The skill offers to enable silent runs; + # if the user declines, it records `degraded` (proceed WITHOUT silent runs) and the + # gate still clears. Downside recorded: the daily digest / Teams reminders / browser + # checks will PROMPT when Scout isn't focused and can stall until the user opens Scout. + opt_out: true + + # ---- ATTEST: the engineer confirms (Scout cannot fully prove these) ---- + # `detail` is the table-cell text and MAY contain inline markdown links + # (they render fine inside Scout table cells). + - id: adx_access + label: "ADX release dashboard" + text: "You can query the ADX release-telemetry cluster (Scout verifies via Kusto)" + detail: "Scout verifies you can query the [ADX release dashboard](https://dataexplorer.azure.com/dashboards/ab7abd7e-c36f-4ff0-88a7-aaa3ec014bd7)'s cluster." + verify: auto + source: scout # the SKILL queries Kusto (the Python engine can't reach the MCP) + verifier: kusto_access + cluster_uri: "https://idsharedeus2.eastus2.kusto.windows.net" + database: "d496be22d62a46b0a3cf67ea2e736fd8" + + - id: play_console_access + label: "Play Console access" + text: "You can open the Play Console app dashboard (sign in and confirm it loads)" + detail: "Open the [Play Console dashboard](https://play.google.com/console/u/0/developers/6720847872553662727/app/4972501392087484764/app-dashboard) and confirm it loads." + verify: attest + + - id: oncall_now + label: "Not on-call now" + text: "You are NOT currently the Android on-call (Scout verifies via ICM)" + detail: "Scout verifies you are not the current Android [on-call](https://aka.ms/androidoncall) for Auth Client Android Shield." + verify: auto + source: scout # the SKILL checks ICM (the Python engine can't reach the MCP) + verifier: oncall + team_id: 78848 + team_name: "Auth Client Android Shield" + + - id: oncall_window + label: "Free during release window" + text: "You are NOT scheduled on-call during the release window" + detail: "Confirm you are NOT scheduled Android [on-call](https://aka.ms/androidoncall) during the release window." + verify: attest + window_start_anchor: "CCD-7" # window opens 7 days before Code Complete + window_end_anchor: "CCD+14" # …through 14 days after + + - id: saw_ame + label: "SAW + AME" + text: "SAW machine and AME account are accessible — log into SAW, confirm the desktop loads, and sign in with your AME credentials" + detail: "Log into [SAW](https://aka.ms/saw), confirm the desktop loads, and sign in with your AME credentials." + verify: attest + + - id: yubikey + label: "YubiKey in hand" + text: "YubiKey in hand (no YubiKey? pick one up at Studio A 1909, Bldg 34/35 cafe, or order a DSR USB-C thumbdrive)" + detail: "YubiKey in hand — no YubiKey? Studio A 1909, Bldg 34/35 cafe, or [order a DSR USB-C](https://cloudmfa-support.azurewebsites.net/SecurityKeyServices/SecurityKey)." + verify: attest diff --git a/release-agent/config/requirements.yaml b/release-agent/config/requirements.yaml new file mode 100644 index 00000000..28e4942c --- /dev/null +++ b/release-agent/config/requirements.yaml @@ -0,0 +1,114 @@ +# Release Orchestrator — external dependency manifest (single source of truth). +# +# Lists everything the tool needs on a machine, so a DIFFERENT release engineer +# can verify their setup is complete. `bootstrap.ps1` reads this file and checks +# each entry (or, for MCP servers, prints them for the engineer to confirm in Scout). +# +# KEEP THIS UPDATED: whenever a feature or phase agent introduces a new dependency +# (a CLI, a Python package, or an MCP server), add it here in the same change. +# +# Entry fields: +# id unique short id +# name human name +# type cli | python | host | mcp +# required_by which part(s) of the tool need it (traceability) +# check shell command that succeeds (exit 0) when present [cli/python/host] +# install how to install it (shown when the check fails) [cli/python/host] +# note guidance (used for `mcp` entries — not shell-checkable) + +requirements: + - id: scout + name: "Microsoft Scout (host app)" + type: host + required_by: [everything — the skill runs inside Scout] + # Detected by the presence of the Scout profile dir (~/.scout). This is a soft + # check surfaced by `infra`; if absent, the engineer must install Scout FIRST + # (MCP servers can't be registered without it). + check: "cmd /c \"if exist \"%USERPROFILE%\\.scout\" (exit 0) else (exit 1)\"" + install: "Install Microsoft Scout first, then re-run bootstrap. (Get it from your team's Scout distribution / the internal Scout install page.)" + + - id: python + name: "Python 3.9+" + type: host + required_by: [engine] + check: "python --version" + install: "Install Python 3.9 or newer from https://www.python.org/downloads/" + + - id: pyyaml + name: "PyYAML" + type: python + required_by: [engine (reads config/*.yaml)] + check: "python -c \"import yaml\"" + install: "python -m pip install pyyaml" + + - id: azure-cli + name: "Azure CLI (az)" + type: cli + required_by: [readiness.build_access] + check: "az version" + install: "https://learn.microsoft.com/cli/azure/install-azure-cli" + + - id: azure-devops-ext + name: "Azure CLI 'azure-devops' extension" + type: cli + required_by: [readiness.build_access] + check: "az extension show --name azure-devops" + install: "az extension add --name azure-devops" + + - id: az-login + name: "Signed in to Azure DevOps (az login)" + type: cli + required_by: [readiness.build_access] + check: "az account show" + install: "az login (must have access to the release build definitions)" + + - id: agency-cli + name: "Agency CLI (provides the ICM MCP server)" + type: cli + required_by: [readiness.oncall_now, mcp.icm] + check: "cmd /c \"%APPDATA%\\agency\\CurrentVersion\\agency.exe --version\"" + install: "Install the Agency platform: https://aka.ms/agency (provides agency.exe under %APPDATA%\\agency)" + +# MCP servers the skill needs inside Scout. Unlike CLIs these live in Scout's own +# config (~/.scout/m-mcp-servers.json) and load at startup — so `bootstrap.ps1` +# REGISTERS any missing ones (create-if-absent, backing up the file first) and then +# tells the engineer to RESTART Scout for them to load. Each entry: +# id short id +# name human name +# scout_key the key to use under "servers" in m-mcp-servers.json +# provider a shell check that the launcher exists (so we don't register a broken server) +# command absolute launcher path (supports %APPDATA% etc.; expanded at register time) +# args launcher args +# required_by which part(s) of the tool need it +# note guidance +mcp_servers: + - id: icm + name: "ICM MCP server (on-call / incidents)" + scout_key: icm + provider: "cmd /c \"%APPDATA%\\agency\\CurrentVersion\\agency.exe --version\"" + command: "%APPDATA%\\agency\\CurrentVersion\\agency.exe" + args: ["mcp", "icm"] + required_by: [readiness.oncall_now, on-call lookups] + note: "Provided by the Agency CLI. After registering, RESTART Scout so it loads; then Scout auto-discovers the ICM tools." + + - id: kusto + name: "Kusto / ADX MCP server (query telemetry)" + scout_key: kusto + provider: "cmd /c \"%APPDATA%\\agency\\CurrentVersion\\agency.exe --version\"" + command: "%APPDATA%\\agency\\CurrentVersion\\agency.exe" + args: ["mcp", "kusto"] + # Multi-cluster: infra appends '--known-services ' built from the + # `kusto_clusters` list below, so ALL our clusters are queryable through one + # MCP. Add a cluster there (data only) to make it available — no code change. + known_services_from: kusto_clusters + required_by: [monitoring/ADX dashboards, Play vitals, crash metrics, readiness.adx_access] + note: "Provided by the Agency CLI (uvx microsoft-fabric-rti-mcp). After registering, RESTART Scout. Add clusters under kusto_clusters." + +# Kusto/ADX clusters the tool queries (data — infra wires these into the kusto MCP +# via --known-services). Add an entry per cluster we use. +kusto_clusters: + - service_uri: "https://idsharedeus2.eastus2.kusto.windows.net" + default_database: "d496be22d62a46b0a3cf67ea2e736fd8" + description: "ID shared EastUS2 (idsharedeus2) — Android release telemetry" + + diff --git a/release-agent/config/schedule.yaml b/release-agent/config/schedule.yaml new file mode 100644 index 00000000..5708fdf6 --- /dev/null +++ b/release-agent/config/schedule.yaml @@ -0,0 +1,20 @@ +# Where the Code Complete Date (CCD) comes from — the system of record. +# +# The orchestrator SEEDS CCD from this pipeline at `init`, and when you change +# CCD (`set-ccd`) or skip the release (`skip-release`) it WRITES BACK to these +# same variables — gated (explicit --confirm) and audited (reason + event log). +# +# CCD rule (mirrors the pipeline's own YAML): +# * override_variable, if set AND its month == the release month -> use it +# * otherwise -> 2nd Wednesday of the month +# +# Per-phase anchors (e.g. Phase 0 opens CCD-7) live next to each phase in +# phases.yaml as `anchor: "CCD-7"`. This file only says WHERE CCD comes from. + +ccd_source: + pipeline_id: 3038 + org: "https://identitydivision.visualstudio.com" + project: "Engineering" + name: "Code Complete Calendar Checker" + override_variable: "overrideCodeCompleteDate" # YYYY-MM-DD, month-scoped + skip_variable: "skipRelease" # any non-empty value suppresses the release diff --git a/release-agent/orchestrator/__init__.py b/release-agent/orchestrator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release-agent/orchestrator/cli.py b/release-agent/orchestrator/cli.py new file mode 100644 index 00000000..49f7e464 --- /dev/null +++ b/release-agent/orchestrator/cli.py @@ -0,0 +1,53 @@ +"""Release Orchestrator — CLI entry point (thin assembler). + +The interface the /release-agent skill calls. This file only wires the parser and +dispatches; the command handlers live in `orchestrator/commands/` (one module per +domain) and shared plumbing in `orchestrator/cli_common.py`. + + python -m orchestrator.cli [options] + +State lives in //release-state.json (gitignored). +Config is release-agent/config/*.yaml. +""" +from __future__ import annotations +import argparse +import os +import sys + +# Force UTF-8 stdout so status glyphs don't crash under Windows cp1252 when piped. +try: + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") +except Exception: + pass + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) # release-agent/ +sys.path.insert(0, ROOT) + +from orchestrator import cli_common as C +from orchestrator.commands import REGISTRARS + + +def build_parser(): + p = argparse.ArgumentParser(prog="release-agent", + description="Release Orchestrator backbone (X4+X5).") + p.add_argument("--config", default=C.DEFAULT_CONFIG) + p.add_argument("--runs-root", default=C.DEFAULT_RUNS_ROOT) + # NOTE: --as-of is defined per-command (status/next/approve/deny/done/resume/notify), + # where it must appear AFTER the subcommand. It is intentionally NOT a global flag: + # argparse lets a subparser's own --as-of silently clobber a global one, which is a + # footgun. Commands that don't take a simulated clock simply omit it. + sub = p.add_subparsers(dest="cmd", required=True) + for register in REGISTRARS: + register(sub) + return p + + +def main(argv=None): + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/release-agent/orchestrator/cli_common.py b/release-agent/orchestrator/cli_common.py new file mode 100644 index 00000000..ade5a53b --- /dev/null +++ b/release-agent/orchestrator/cli_common.py @@ -0,0 +1,147 @@ +"""Shared CLI plumbing for the Release Orchestrator command modules. + +The CLI is split into a thin assembler (`cli.py`) plus one module per command +domain under `commands/`. This module holds the pieces those command modules +share: path resolution, state/orchestrator loading, the event log, user-facing +emit (print + auto-log), and the small render helpers used when advancing. + +Everything here takes explicit parameters (runs_root / release / config) rather +than the argparse namespace, so the helpers are decoupled from the parser and +easy to reuse and test. +""" +from __future__ import annotations + +import os + +from orchestrator.state import ReleaseState +from orchestrator.engine import Orchestrator +from orchestrator import discovery, render, schedule +from orchestrator.eventlog import EventLog +from tools import checks +import yaml as _yaml + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) # release-agent/ +DEFAULT_CONFIG = os.path.join(ROOT, "config", "phases.yaml") +SCHEDULE_CONFIG = os.path.join(ROOT, "config", "schedule.yaml") +REQUIREMENTS_CONFIG = os.path.join(ROOT, "config", "requirements.yaml") +# runs live OUTSIDE release-agent/, in android-complete/.release-runs (gitignored) +DEFAULT_RUNS_ROOT = os.path.join(os.path.dirname(ROOT), ".release-runs") + + +# ---- paths / state ---- +def state_path(runs_root: str, release: str) -> str: + return os.path.join(runs_root, release, "release-state.json") + + +def load_state(runs_root: str, release: str) -> ReleaseState: + return ReleaseState.load(state_path(runs_root, release)) + + +def save_state(st: ReleaseState, runs_root: str, release: str) -> None: + st.save(state_path(runs_root, release)) + + +def parse_as_of(args): + """The simulated clock from --as-of (None ⇒ engine uses today).""" + s = getattr(args, "as_of", None) + return schedule.parse_date(s) if s else None + + +def load_orch(runs_root: str, release: str, config: str, as_of=None): + """Load state + build an Orchestrator wired to the --as-of clock.""" + st = load_state(runs_root, release) + return st, Orchestrator(config, st, as_of=as_of) + + +# ---- config ---- +def ccd_source() -> dict: + """Where CCD comes from (pipeline coords) — from config/schedule.yaml.""" + try: + with open(SCHEDULE_CONFIG, "r", encoding="utf-8") as fh: + return (_yaml.safe_load(fh) or {}).get("ccd_source", {}) or {} + except OSError: + return {} + + +# ---- event log / emit ---- +def elog(runs_root: str, release: str) -> EventLog: + return EventLog(runs_root, release) + + +def emit(runs_root: str, release: str, text: str, kind: str = "message", options=None): + """Print a user-facing block AND auto-log it as scout output, so the log + always captures 'what was shown' without relying on the skill/LLM to journal.""" + print(text) + try: + elog(runs_root, release).scout_said(text, kind=kind, options=options) + except Exception: + pass + + +# ---- advancing the loop (shared by next / approve / done) ---- +TAGS = {"ran": "[ok]", "gate": "[gate]", "reminder": "[action]", "scheduled": "[scheduled]", + "complete": "[done]", "idle": "[--]", "readiness": "[entry-gate]", + "blocked": "[BLOCKED]", "halted": "[HALTED]"} + + +def log_actions(el: EventLog, actions): + """Record engine actions as events (step_ran / gate_hold / complete / blocked).""" + events = { + "ran": "step_ran", "gate": "gate_hold", "reminder": "reminder_hold", + "scheduled": "scheduled_hold", "readiness": "readiness_hold", + "blocked": "blocked_hold", "halted": "halted_hold", "complete": "release_complete", + } + for a in actions: + name = events.get(a.kind) + if not name: + continue + if a.kind in ("ran", "gate", "reminder", "scheduled"): + el.log(name, phase=a.phase, step=a.step, name=a.name) + else: + el.log(name) + + +def advance_block(actions, orch, lead=None) -> str: + """The canonical 'what happened + new status' block for advance commands.""" + out = list(lead or []) + for a in actions: + out.append(f" {TAGS.get(a.kind, '-')} {a.message}") + out.append("\n" + render.status_view(orch.status_report())) + return "\n".join(out) + + +# ---- CCD / pipeline helpers ---- +def refresh_conflict(st: ReleaseState) -> bool: + """Best-effort: re-read the pipeline override and refresh st.ccd_conflict + (a pipeline date that differs from our stored CCD). Returns True if the state + changed (caller should save). Silent on any read failure — never blocks.""" + if not st.ccd: + return False + src = ccd_source() + if not src.get("pipeline_id"): + return False + ok, val, _ = checks.read_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["override_variable"]) + if not ok: + return False + conflict = schedule.pipeline_conflict(st.release_id, val, st.ccd) + new = conflict.isoformat() if conflict else None + if new != st.ccd_conflict: + st.ccd_conflict = new + return True + return False + + +def write_ccd_var(src: dict, value: str): + """Write the CCD override variable on the pipeline. Returns CheckResult.""" + return checks.set_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["override_variable"], value) + + +def resolve_release_id(runs_root: str, release): + """Return an explicit release id or discover the active one (or None).""" + if release: + return release + rel = discovery.resolve(runs_root, None).get("release") + return rel["release_id"] if rel else None diff --git a/release-agent/orchestrator/commands/__init__.py b/release-agent/orchestrator/commands/__init__.py new file mode 100644 index 00000000..c6417e74 --- /dev/null +++ b/release-agent/orchestrator/commands/__init__.py @@ -0,0 +1,21 @@ +"""Command modules for the Release Orchestrator CLI. + +Each module in this package owns one domain of commands. A module exposes a +`register(subparsers)` function that adds its subparser(s) and wires each to its +handler via `set_defaults(func=...)`. `cli.py` imports REGISTRARS and calls each +one, so adding a command is a localized change (new/edited module only). +""" +from . import release, readiness, pipeline, notify, infra_cmd, automation, logs, lockdown, notice + +# Order controls how subcommands appear in --help. +REGISTRARS = [ + release.register, + readiness.register, + pipeline.register, + notify.register, + lockdown.register, + notice.register, + logs.register, + automation.register, + infra_cmd.register, +] diff --git a/release-agent/orchestrator/commands/automation.py b/release-agent/orchestrator/commands/automation.py new file mode 100644 index 00000000..63786730 --- /dev/null +++ b/release-agent/orchestrator/commands/automation.py @@ -0,0 +1,54 @@ +"""Automation registry command: register / list / deregister provisioned Scout +automations so they can be torn down cleanly at release close.""" +from __future__ import annotations +import json as _json + +from orchestrator.registry import AutomationRegistry + + +def cmd_automation(args): + """Track Scout automations the orchestrator provisions, so they can be torn + down at release close. This only records ids — the skill does the actual + Scout create/delete via m_create_automation / m_delete_automation.""" + reg = AutomationRegistry(args.runs_root) + if args.action == "register": + if not (args.id and args.name): + print("register needs --id and --name.") + return 1 + e = reg.register(args.id, args.name, release=args.release, + shared=args.shared, purpose=args.purpose or "") + where = "shared" if e["scope"] == "shared" else f"release {e['release']}" + print(f"Registered automation {e['id']} ({where}): {e['name']}") + return 0 + if args.action == "deregister": + if not args.id: + print("deregister needs --id.") + return 1 + print("Deregistered." if reg.deregister(args.id) else "No such automation id in registry.") + return 0 + # list + items = reg.list(release=args.release, scope=(args.scope or None)) + if args.json: + print(_json.dumps(items, indent=2)) + return 0 + if not items: + print("No automations registered." if args.release is None + else f"No automations registered for release {args.release}.") + return 0 + for e in items: + where = "shared" if e.get("scope") == "shared" else (e.get("release") or "?") + print(f" {e['id']} [{where}] {e['name']} — {e.get('purpose','')}") + return 0 + + +def register(sub): + au = sub.add_parser("automation", help="Track provisioned automations (register/list/deregister) for teardown") + au.add_argument("action", choices=["register", "list", "deregister"]) + au.add_argument("--id", default=None, help="Scout automation id") + au.add_argument("--name", default="", help="Automation name (for register)") + au.add_argument("--release", default=None, help="Release scope (omit + --shared for machine-wide)") + au.add_argument("--shared", action="store_true", help="Mark as shared/persistent (not torn down per release)") + au.add_argument("--scope", default=None, choices=["shared", "release"], help="Filter list by scope") + au.add_argument("--purpose", default="", help="Short description") + au.add_argument("--json", action="store_true") + au.set_defaults(func=cmd_automation) diff --git a/release-agent/orchestrator/commands/infra_cmd.py b/release-agent/orchestrator/commands/infra_cmd.py new file mode 100644 index 00000000..38df422e --- /dev/null +++ b/release-agent/orchestrator/commands/infra_cmd.py @@ -0,0 +1,53 @@ +"""Infrastructure preflight command: check CLIs + register/verify MCP servers in +Scout. Named infra_cmd to avoid clashing with the orchestrator.infra module.""" +from __future__ import annotations +import json as _json + +from orchestrator import infra +from orchestrator import cli_common as C + + +def cmd_infra(args): + """Infrastructure preflight: check CLI/host deps and register + verify the + MCP servers the skill needs in Scout. Run before the tool-level requirements. + Registers missing MCP servers into Scout's config (backup first) unless + --no-register; --json for machine output.""" + report = infra.run(C.REQUIREMENTS_CONFIG, register=not getattr(args, "no_register", False)) + if getattr(args, "json", False): + print(_json.dumps(report, indent=2)) + return 0 if report["ok"] else 1 + print("Infrastructure preflight") + if not report.get("scout_present", True): + print(" ⛔ Microsoft Scout not detected (~/.scout missing).") + scout_req = next((r for r in report["requirements"] if r["id"] == "scout"), None) + if scout_req and scout_req.get("install"): + print(f" install: {scout_req['install']}") + print(" Install Scout FIRST, then re-run — MCP servers can't be registered without it.") + print(" CLIs / host:") + for r in report["requirements"]: + mark = "OK" if r["ok"] else "MISSING" + print(f" [{mark}] {r['name']}") + if not r["ok"] and r["install"]: + print(f" install: {r['install']}") + print(" MCP servers (Scout config):") + if not report["mcp_servers"]: + print(" (none required)") + for m in report["mcp_servers"]: + label = {"present": "OK", "registered": "REGISTERED", "would_register": "MISSING", + "provider_missing": "PROVIDER MISSING", "launcher_missing": "LAUNCHER MISSING", + "scout_missing": "SCOUT NOT INSTALLED"}.get(m["status"], m["status"].upper()) + print(f" [{label}] {m['name']} — {m['detail']}") + if report["restart_needed"]: + print("\n ⚠ RESTART Scout to load newly-registered MCP server(s).") + if not report["ok"]: + print("\n Some infrastructure is missing — resolve the items above, then re-run.") + else: + print("\n Infrastructure OK.") + return 0 if report["ok"] else 1 + + +def register(sub): + inf = sub.add_parser("infra", help="Infrastructure preflight: check CLIs + register/verify MCP servers in Scout") + inf.add_argument("--no-register", action="store_true", help="Only report; don't register missing MCP servers") + inf.add_argument("--json", action="store_true") + inf.set_defaults(func=cmd_infra) diff --git a/release-agent/orchestrator/commands/lockdown.py b/release-agent/orchestrator/commands/lockdown.py new file mode 100644 index 00000000..1668c86d --- /dev/null +++ b/release-agent/orchestrator/commands/lockdown.py @@ -0,0 +1,88 @@ +"""Lockdown / CCOA overlap check (Phase-0 step `lockdown`, S1). + +The CCOA "No-Fly Zones" source is an AAD-gated web app, so the SKILL scrapes it +via the authenticated browser and passes the periods here as JSON. This command +decides overlap DETERMINISTICALLY (not the LLM) and records the step result: + * no Production-env CCOA overlaps the release window -> step passes. + * one or more overlap -> step holds for the owner (who shifts CCD via set-ccd). +""" +from __future__ import annotations +import json as _json + +from orchestrator import cli_common as C +from orchestrator import schedule + + +def _load_lockdown_cfg() -> dict: + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight", "lockdown") + + +def overlapping_periods(win_start, win_end, periods, blocking_env="Production"): + """Pure overlap rule. `periods` is a list of dicts with keys name, environment, + start (date), end (date). Returns those whose Environment contains + `blocking_env` (case-insensitive) AND whose [start,end] intersects the window + [win_start,win_end]. Banner-only advisories (no Production) are excluded.""" + hits = [] + for p in periods: + if blocking_env.lower() not in (p.get("environment") or "").lower(): + continue + s, e = p.get("start"), p.get("end") + if s is None or e is None: + continue + if s <= win_end and e >= win_start: # ranges intersect + hits.append(p) + return hits + + +def cmd_check_lockdown(args): + st = C.load_state(args.runs_root, args.release) + if not st.ccd: + print("No CCD set for this release — cannot compute the release window.") + return 1 + cfg = _load_lockdown_cfg() + blocking_env = cfg.get("blocking_environment", "Production") + ccd = schedule.parse_date(st.ccd) + win_start = schedule.anchor_date(ccd, cfg.get("window_start_anchor", "CCD-7")) + win_end = schedule.anchor_date(ccd, cfg.get("window_end_anchor", "CCD+14")) + + try: + raw = _json.loads(args.periods_json or "[]") + except ValueError: + print("Could not parse --periods-json (expected a JSON array).") + return 1 + periods = [] + for p in raw if isinstance(raw, list) else []: + s, e = schedule.parse_date(p.get("start")), schedule.parse_date(p.get("end")) + periods.append({"name": p.get("name", "?"), + "environment": p.get("environment", ""), "start": s, "end": e}) + + hits = overlapping_periods(win_start, win_end, periods, blocking_env) + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + window = f"{win_start.isoformat()}..{win_end.isoformat()}" + if not hits: + detail = (f"No {blocking_env} CCOA lockdown overlaps the release window " + f"({window}). Checked {len(periods)} period(s).") + orch.record_scout_step("preflight", "lockdown", "pass", detail) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, f"[ok] Lockdown check: {detail}", kind="lockdown") + return 0 + + listed = "; ".join( + f"{h['name']} ({h['start'].isoformat()}..{h['end'].isoformat()})" for h in hits) + detail = (f"{len(hits)} {blocking_env} CCOA lockdown(s) overlap the release window " + f"({window}): {listed}. Shift CCD past the lockdown (set-ccd) if needed.") + orch.record_scout_step("preflight", "lockdown", "attention", detail) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, f"[attention] Lockdown overlap — {detail}", kind="lockdown") + return 0 + + +def register(sub): + cl = sub.add_parser("check-lockdown", + help="Decide CCOA lockdown overlap from scraped periods and record the step") + cl.add_argument("--release", required=True) + cl.add_argument("--periods-json", required=True, + help='JSON array of {name, environment, start, end} (dates YYYY-MM-DD, UTC)') + cl.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + cl.set_defaults(func=cmd_check_lockdown) diff --git a/release-agent/orchestrator/commands/logs.py b/release-agent/orchestrator/commands/logs.py new file mode 100644 index 00000000..6c470832 --- /dev/null +++ b/release-agent/orchestrator/commands/logs.py @@ -0,0 +1,64 @@ +"""Event-log commands: log (show/analyze) and journal (record interaction).""" +from __future__ import annotations +import json as _json + +from orchestrator.eventlog import EventLog, summarize +from orchestrator import cli_common as C + + +def cmd_log(args): + """Show or analyze this release's event log (per-release only).""" + el = EventLog(args.runs_root, args.release) + events = el.read(args.limit) + if args.analyze: + print(_json.dumps(summarize(events), indent=2)) + return 0 + if args.json: + print(_json.dumps(events, indent=2)) + return 0 + if not events: + print("No events logged yet.") + return 0 + for e in events: + src = e.get("source", "engine") + loc = f" {e['phase']}/{e.get('step','')}" if e.get("phase") else "" + extra = "" + if e.get("driver"): + extra += f" driver=\"{e['driver']}\"" + if e.get("text"): + t = e["text"].replace("\n", " ") + extra += f" \"{t[:80]}{'…' if len(t) > 80 else ''}\"" + if e.get("choice"): + extra += f" choice={e['choice']}" + print(f" {e['ts']} {src:<6} {e.get('actor','?'):<10} {e['event']}{loc}{extra}") + return 0 + + +def cmd_journal(args): + """Record an INTERACTION event (what Scout showed / what the user chose). + The skill calls this so the per-release log captures the real conversation + for debugging. Best-effort; never affects the flow.""" + el = C.elog(args.runs_root, args.release) + if args.source == "scout": + el.scout_said(args.text or "", kind=args.kind or "message", options=args.option or None) + else: + el.user_said(args.text or "", kind=args.kind or "input", choice=args.choice or None) + return 0 + + +def register(sub): + lg = sub.add_parser("log", help="Show or analyze this release's event log") + lg.add_argument("--release", required=True) + lg.add_argument("--analyze", action="store_true", help="Print a rolled-up summary") + lg.add_argument("--limit", type=int, default=None) + lg.add_argument("--json", action="store_true") + lg.set_defaults(func=cmd_log) + + jn = sub.add_parser("journal", help="Record an interaction event (scout output / user input)") + jn.add_argument("--release", required=True) + jn.add_argument("--source", required=True, choices=["scout", "user"]) + jn.add_argument("--text", default="", help="What was shown / said") + jn.add_argument("--kind", default="", help="e.g. prompt, checklist, message, choice, input") + jn.add_argument("--choice", default="", help="For user: the option id/label chosen") + jn.add_argument("--option", action="append", help="For scout: an option presented (repeatable)") + jn.set_defaults(func=cmd_journal) diff --git a/release-agent/orchestrator/commands/notice.py b/release-agent/orchestrator/commands/notice.py new file mode 100644 index 00000000..05779faa --- /dev/null +++ b/release-agent/orchestrator/commands/notice.py @@ -0,0 +1,253 @@ +"""Early code-complete notice (Phase-0 step `notice`, S0) + generic step recorder. + +Sending email needs the WorkIQ MCP (a skill-layer capability the deterministic +engine can't reach), so `notice` is a scout-assisted step: + + 1. `prepare-notice` (here, deterministic) fills the local template with the + release's CCD/owner and resolves recipients — DRY-RUN redirects every mail to + the release owner; a LIVE release uses the real recipients from preflight.yaml. + It prints {subject, body, recipients, dry_run, ...} for the skill to send. + 2. the skill sends it via workiq_send_email, then calls `record-step` to mark + the step done (or attention on failure). + +`record-step` is a generic recorder for ANY scout-assisted phase step. +""" +from __future__ import annotations +import json as _json +import os + +from orchestrator import cli_common as C +from orchestrator import schedule + +_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Fixed external link used inside the notice body (see EXTERNAL-REFERENCES.md). +HOTFIX_GUIDE_URL = ("https://eng.ms/docs/microsoft-security/identity/" + "entra-developer-application-platform/auth-client/" + "microsoft-authenticator/microsoft-authenticator/release/" + "cherry-pick-to-hotfix-guidelines") + + +def _load_notice_cfg() -> dict: + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight", "notice") + + +def _ordinal(n: int) -> str: + if 11 <= (n % 100) <= 13: + return f"{n}th" + return f"{n}{ {1: 'st', 2: 'nd', 3: 'rd'}.get(n % 10, 'th') }" + + +def _parse_template(text: str, variant: str): + """Pull the (subject, body) for a variant from the delimited template file. + Sections are marked '===INITIAL:SUBJECT===' / '===INITIAL:BODY===' etc.""" + key = variant.upper() + marks = {"subject": f"==={key}:SUBJECT===", "body": f"==={key}:BODY==="} + out = {} + for field, mark in marks.items(): + if mark not in text: + return None + after = text.split(mark, 1)[1] + # body runs until the next '===...===' marker or EOF + end = after.find("\n===") + out[field] = (after[:end] if end != -1 else after).strip("\n") + return out["subject"].strip(), out["body"] + + +def _fill(s: str, ctx: dict) -> str: + for k, v in ctx.items(): + s = s.replace("{" + k + "}", str(v)) + return s + + +def _esc(s: str) -> str: + return (str(s or "").replace("&", "&").replace("<", "<") + .replace(">", ">")) + + +def _notice_html(variant: str, ctx: dict) -> str: + """Email-safe HTML notice: clean anchor for the hotfix guide + a real table + (inline styles, Outlook-friendly). Same content as the markdown body.""" + date_line = ("**Today**" if variant == "update" + else f"{_esc(ctx['ccd_long'])}.") + if variant == "update": + date_line = "Today" + owner = _esc(ctx["owner"]) + return f"""\ +
+

Hi everyone,

+

This is a reminder that the Microsoft Android Authenticator app and Broker + libraries code complete date for the {_esc(ctx['month'])} release is {date_line}

+

Any check-ins made after code complete will require following + the hotfix cherry-pick guide + and EM approval.

+ + + + + + + + + + + +
MonthCode Complete DateAndroid Release Owner
{_esc(ctx['month'])}{_esc(ctx['ccd_date'])} + Primary (Release Owner — covers Broker + Auth App): @{_esc(ctx['owner_at'])}
+

Thank you,

+

{owner}

+
""" + + +def cmd_prepare_notice(args): + st = C.load_state(args.runs_root, args.release) + if not st.ccd: + print(_json.dumps({"error": "no CCD set for this release"})) + return 1 + cfg = _load_notice_cfg() + variant = getattr(args, "variant", None) or cfg.get("variant", "initial") + tpl_path = os.path.join(_ROOT, cfg.get("template", "templates/early-code-complete-notice.md")) + try: + with open(tpl_path, "r", encoding="utf-8") as fh: + parsed = _parse_template(fh.read(), variant) + except OSError: + print(_json.dumps({"error": f"template not found: {tpl_path}"})) + return 1 + if not parsed: + print(_json.dumps({"error": f"variant '{variant}' not in template"})) + return 1 + subject_tpl, body_tpl = parsed + + ccd = schedule.parse_date(st.ccd) + owner_email = st.owner_email or "" + ctx = { + "month": ccd.strftime("%B"), + "ccd_long": f"{ccd.strftime('%A, %B')} {_ordinal(ccd.day)}, {ccd.year}", + "ccd_date": ccd.strftime("%m/%d/%Y"), + "owner": st.owner_name or owner_email or "the release owner", + "owner_at": (owner_email.split("@")[0] if owner_email else "release-owner"), + } + subject = _fill(subject_tpl, ctx) + body = _fill(body_tpl, ctx) + html = _notice_html(variant, ctx) + + # RECIPIENTS: dry-run → owner only (safe); live → configured real list. + if st.dry_run: + recipients = [owner_email] if owner_email else [] + note = "dry-run: redirected to release owner" + else: + recipients = list(cfg.get("recipients", [])) + note = "live recipients" + subject_out = (f"[DRY-RUN → owner] {subject}" if st.dry_run else subject) + + print(_json.dumps({ + "step": "notice", "release": args.release, "dry_run": st.dry_run, + "subject": subject_out, "body": body, "html": html, "recipients": recipients, + "recipients_note": note, + })) + return 0 + + +def _reminder_ctx(st): + """Shared date/owner context for the Teams reminders.""" + ccd = schedule.parse_date(st.ccd) + owner_email = st.owner_email or "" + ccd7 = schedule.anchor_date(ccd, "CCD-7") + return { + "month": ccd.strftime("%B"), + "ccd_long": f"{ccd.strftime('%A, %B')} {_ordinal(ccd.day)}, {ccd.year}", + "ccd_date": ccd.strftime("%m/%d/%Y"), + "ccd7_date": ccd7.strftime("%m/%d/%Y"), + "owner": st.owner_name or owner_email or "the release owner", + }, owner_email + + +def _reminder_cfg(section: str) -> dict: + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight", section) + + +def _reminder_payload(step, args, st, cfg, html, owner_email): + """Resolve the Teams target: DRY-RUN → owner's own chat; LIVE → group chat.""" + if st.dry_run: + html = ("

[DRY-RUN → owner] this would go to the " + f"{_esc(cfg.get('live_chat_name', 'Android Core Team'))} group chat.

" + html) + return {"step": step, "release": args.release, "dry_run": True, + "content": html, "content_type": "html", + "send_to": "owner", "owner_email": owner_email, "chat_id": None, + "target_note": "dry-run: send to the release owner's own Teams chat"} + return {"step": step, "release": args.release, "dry_run": False, + "content": html, "content_type": "html", + "send_to": "group", "owner_email": owner_email, + "chat_id": cfg.get("live_chat_id"), + "target_note": cfg.get("live_chat_name", "Android Core Team")} + + +def _flight_reminder_html(ctx: dict, links: dict) -> str: + """Teams-friendly HTML for the combined feature-owner reminders.""" + vg = links.get("variable_group", "#") + pm = links.get("premortem_example", "#") + loc = links.get("localization", "#") + ecs = links.get("ecs_flight_history", "#") + return f"""\ +

Flight & String Reminders — {_esc(ctx['month'])} Release

+

Hi Android Core Team, four reminders as we approach code complete ({_esc(ctx['ccd_long'])}):

+
    +
  1. [Broker] Update local flights. Feature owners — please update your local flights in the release variable group. The release engineer will not update these directly; verify your values are current before we proceed. (Variable group)
    2. +
  2. [Broker & Auth App] Flight pre-mortem docs. It is each Feature Owner's sole responsibility to ensure every flight shipping with a default value of true has a flight pre-mortem doc — explaining the change's purpose and how we plan to monitor it after the release hits PROD. (Example pre-mortem doc)
    3. +
  3. [Auth App] Merge user-facing strings by today (CCD−7). It is each Feature Owner's sole responsibility to ensure every Authenticator PR with a new or modified user-facing string is merged by today ({_esc(ctx['ccd7_date'])}). Strings landing inside the 1-week window are not guaranteed to be localized — call out any late strings and escalate to the team lead if not resolved by end of day. (Localization Instructions)
    4. +
  4. [Auth App] Feature-flag freeze & default-OFF review. Review all features added since the last release and ensure they are default OFF; any feature rolling out default ON requires explicit Team Lead / Engineering Manager approval documented in the release wiki, and the release should be blocked if such approval is missing. No feature work is allowed after Code Complete (CCD) — all feature-flag changes must land before CCD, and any flag changes after CCD require the same cherry-pick approval process as code changes. Review EcsFlight.kt history since the last Code Complete to verify compliance. (EcsFlight.kt history)
    5. +
+

Thanks,
{_esc(ctx['owner'])}

""" + + +def cmd_prepare_flight_reminder(args): + """Build the combined 3-in-1 flight & string reminder Teams message and resolve + its target. DRY-RUN → the release owner's own Teams chat; LIVE → the Android + Core Team group chat. Prints JSON for the skill to send via workiq_send_chat_message.""" + st = C.load_state(args.runs_root, args.release) + if not st.ccd: + print(_json.dumps({"error": "no CCD set for this release"})) + return 1 + cfg = _reminder_cfg("flight_reminder") + ctx, owner_email = _reminder_ctx(st) + html = _flight_reminder_html(ctx, cfg.get("links", {}) or {}) + print(_json.dumps(_reminder_payload("flight_reminder", args, st, cfg, html, owner_email))) + return 0 + + +def cmd_record_step(args): + """Generic recorder for a scout-assisted phase step (skill calls this after + doing the out-of-engine work, e.g. sending the notice email).""" + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + act = orch.record_scout_step(args.phase, args.step, args.status, args.detail or "") + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f"[{'ok' if args.status == 'pass' else 'attention'}] {args.step}: {act.message}", + kind="step") + return 0 + + +def register(sub): + pn = sub.add_parser("prepare-notice", + help="Fill the early code-complete notice template and resolve recipients (JSON)") + pn.add_argument("--release", required=True) + pn.add_argument("--variant", default=None, help="initial (default) | update") + pn.set_defaults(func=cmd_prepare_notice) + + pf = sub.add_parser("prepare-flight-reminder", + help="Build the combined flight & string reminder Teams message + target (JSON)") + pf.add_argument("--release", required=True) + pf.set_defaults(func=cmd_prepare_flight_reminder) + + rs = sub.add_parser("record-step", + help="Record a scout-assisted phase step result (pass|attention)") + rs.add_argument("--release", required=True) + rs.add_argument("--phase", default="preflight") + rs.add_argument("--step", required=True) + rs.add_argument("--status", required=True, choices=["pass", "attention"]) + rs.add_argument("--detail", default="") + rs.add_argument("--as-of", default=None) + rs.set_defaults(func=cmd_record_step) diff --git a/release-agent/orchestrator/commands/notify.py b/release-agent/orchestrator/commands/notify.py new file mode 100644 index 00000000..f93b9f39 --- /dev/null +++ b/release-agent/orchestrator/commands/notify.py @@ -0,0 +1,126 @@ +"""Notification + owner commands: notify (daily phase digest) and set-owner.""" +from __future__ import annotations +import json as _json +import os + +from orchestrator.state import ReleaseState +from orchestrator.engine import Orchestrator +from orchestrator import render, schedule +from orchestrator import cli_common as C +from tools import checks + + +def cmd_set_owner(args): + """Set/change the release owner (who reminders are emailed to).""" + st = C.load_state(args.runs_root, args.release) + st.owner_email = (getattr(args, "owner_email", None) or checks.current_az_user()) + if getattr(args, "owner_name", None): + st.owner_name = args.owner_name + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("owner_set", owner=st.owner_email) + if not st.owner_email: + print("Couldn't resolve an owner (no --owner-email and az user unavailable).") + return 1 + who = f"{st.owner_name + ' ' if st.owner_name else ''}{st.owner_email}" + print(f"Release {args.release} owner set to {who}.") + return 0 + + +def cmd_notify(args): + """Emit the daily phase digest IF the active phase is open with outstanding + work, else nothing. Read-only (does NOT advance the flow — use `tick` for that). + De-duped to one per calendar day; --force bypasses; --json prints the mailer + payload {message,subject,owner_email,owner_name,release}.""" + rid = C.resolve_release_id(args.runs_root, args.release) + want_json = getattr(args, "json", False) + if not rid: + if want_json: + print(_json.dumps({"message": "", "html": "", "subject": "", "owner_email": None, + "owner_name": None, "release": None})) + return 0 + payload = _notify_payload(args, rid, advance=False) + if want_json: + print(_json.dumps(payload)) + elif payload["message"]: + print(payload["message"]) + return 0 + + +def _notify_payload(args, rid, advance): + """Shared by `notify` and `tick`. Optionally ADVANCE the flow first + (run_until_gate), then read the state machine and build the once-per-day + digest payload. Returns {message, subject, owner_email, owner_name, release}. + `message` is "" unless a digest is due AND not already sent today (or --force).""" + sp = C.state_path(args.runs_root, rid) + if not os.path.exists(sp): + return {"message": "", "html": "", "subject": "", "owner_email": None, + "owner_name": None, "release": rid} + as_of = C.parse_as_of(args) + if advance: + # Auto-advance: run every agent step that can run, holding at the first + # gate / action-needed. Idempotent — a no-op once holding or not due. + st, orch = C.load_orch(args.runs_root, rid, args.config, as_of) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, rid) + C.log_actions(C.elog(args.runs_root, rid), actions) + else: + st = ReleaseState.load(sp) + orch = Orchestrator(args.config, st, as_of=as_of) + report = orch.status_report() + msg = render.notification(report) + html = render.notification_html(report) + subject = render.notification_subject(report) + today = (as_of or schedule.today()).isoformat() + fresh = bool(msg) and (getattr(args, "force", False) or st.last_notified_date != today) + if fresh: + st.last_notified_date = today + st.save(sp) + try: + C.elog(args.runs_root, rid).log("notified", text=msg, owner=st.owner_email) + except Exception: + pass + return {"message": msg if fresh else "", "html": html if fresh else "", + "subject": subject, "owner_email": st.owner_email, + "owner_name": st.owner_name, "release": rid} + + +def cmd_tick(args): + """One automation heartbeat: discover the active release, ADVANCE it (run the + agent steps that can run, holding at gates/actions), then emit the daily digest + payload for the mailer. Safe to run often — advancing is idempotent and the + digest is de-duped to once per calendar day. This is what the hourly Scout + automation runs so an open phase makes progress even if the 9am tick was missed + (machine off) — the next tick after the machine is on picks it up.""" + rid = C.resolve_release_id(args.runs_root, args.release) + if not rid: + print(_json.dumps({"message": "", "html": "", "subject": "", "owner_email": None, + "owner_name": None, "release": None})) + return 0 + payload = _notify_payload(args, rid, advance=True) + if getattr(args, "json", False): + print(_json.dumps(payload)) + elif payload["message"]: + print(payload["message"]) + return 0 + + +def register(sub): + so = sub.add_parser("set-owner", help="Set/change the release owner (who reminders are emailed to)") + so.add_argument("--release", required=True) + so.add_argument("--owner-email", default=None, help="Owner email (default: signed-in az user)") + so.add_argument("--owner-name", default=None, help="Owner display name (optional)") + so.set_defaults(func=cmd_set_owner) + + nt = sub.add_parser("notify", help="Emit a push line if something needs the user now (else nothing)") + nt.add_argument("--release", default=None, help="Target release; if omitted, discover the active one") + nt.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD) — debug override; default today") + nt.add_argument("--force", action="store_true", help="Bypass de-dup (always emit if actionable)") + nt.add_argument("--json", action="store_true", help="Emit {message,subject,owner_email,owner_name,release} for the mailer") + nt.set_defaults(func=cmd_notify) + + tk = sub.add_parser("tick", help="Automation heartbeat: ADVANCE the active release, then emit the digest payload") + tk.add_argument("--release", default=None, help="Target release; if omitted, discover the active one") + tk.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD) — debug override; default today") + tk.add_argument("--force", action="store_true", help="Bypass the once-per-day digest de-dup") + tk.add_argument("--json", action="store_true", help="Emit {message,subject,owner_email,owner_name,release} for the mailer") + tk.set_defaults(func=cmd_tick) diff --git a/release-agent/orchestrator/commands/pipeline.py b/release-agent/orchestrator/commands/pipeline.py new file mode 100644 index 00000000..004e801d --- /dev/null +++ b/release-agent/orchestrator/commands/pipeline.py @@ -0,0 +1,105 @@ +"""Pipeline-write commands (real production changes to ADO pipeline 3038): +set-ccd and skip-release. Both are gated (preview → --confirm) and audited.""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator import cli_common as C +from tools import checks + + +def cmd_set_ccd(args): + """Change the Code Complete Date. Writes the pipeline override (real change) — + requires --confirm and a --reason. Without --confirm, previews the write.""" + st = C.load_state(args.runs_root, args.release) + src = C.ccd_source() + if not src.get("pipeline_id"): + print("No CCD source configured (config/schedule.yaml).") + return 1 + if not (args.reason and args.reason.strip()): + print("A --reason is required (audited).") + return 1 + + if args.default: + new_ccd, source, value = schedule.default_ccd(args.release), "default", "" + what = f"clear the override → default {new_ccd.isoformat()} (2nd Wednesday)" + else: + d = schedule.parse_date(args.date) + if not d: + print(f"Bad --date '{args.date}' (expected YYYY-MM-DD).") + return 1 + ry, rm = schedule.parse_release_month(args.release) + if (d.year, d.month) != (ry, rm): + print(f"CCD {d.isoformat()} is not in release month {args.release}. The pipeline " + f"override is month-scoped, so a different month wouldn't apply. " + f"Use the release id for that month instead.") + return 1 + new_ccd, source, value = d, "manual", d.isoformat() + what = f"set CCD override → {value}" + + if not args.confirm: + print(f"[preview] Would {what} on pipeline {src['pipeline_id']} " + f"({src['override_variable']}).\n Re-run with --confirm to write it. Reason: {args.reason.strip()}") + return 0 + + res = C.write_ccd_var(src, value) + if not res.ok: + print(f"Failed to write pipeline variable: {res.detail}") + return 1 + st.ccd = new_ccd.isoformat() + st.ccd_source = source + st.ccd_conflict = None # the date is now settled — clear any conflict + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "ccd_changed", value=value or "(default)", source=source, driver=args.reason.strip()) + opens = schedule.anchor_date(new_ccd, "CCD-7").isoformat() + C.emit(args.runs_root, args.release, + f"✅ CCD set to **{st.ccd}** ({source}); pipeline updated. " + f"Phase 0 opens {opens} (CCD-7).\n {res.detail}", kind="ccd") + return 0 + + +def cmd_skip_release(args): + """Suppress the release by setting the pipeline 'skipRelease' switch (real change).""" + st = C.load_state(args.runs_root, args.release) + src = C.ccd_source() + if not (args.reason and args.reason.strip()): + print("A --reason is required (audited).") + return 1 + clearing = bool(getattr(args, "clear", False)) + value = "" if clearing else "skipped" + verb = "clear" if clearing else "set" + if not args.confirm: + print(f"[preview] Would {verb} '{src.get('skip_variable')}' on pipeline " + f"{src.get('pipeline_id')}.\n Re-run with --confirm. Reason: {args.reason.strip()}") + return 0 + res = checks.set_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["skip_variable"], value) + if not res.ok: + print(f"Failed to write pipeline variable: {res.detail}") + return 1 + st.skip_release = not clearing + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "release_skip_cleared" if clearing else "release_skip_set", driver=args.reason.strip()) + msg = ("✅ Release un-skipped — pipeline will trigger normally." + if clearing else + "🛑 Release marked SKIP in the pipeline — the monthly trigger is suppressed until cleared.") + C.emit(args.runs_root, args.release, msg + f"\n {res.detail}", kind="skip_release") + return 0 + + +def register(sub): + sc = sub.add_parser("set-ccd", help="Change the Code Complete Date (writes pipeline override; --confirm)") + sc.add_argument("--release", required=True) + sc.add_argument("--date", default="", help="New CCD (YYYY-MM-DD), must be in the release month") + sc.add_argument("--default", action="store_true", help="Clear the override → 2nd-Wednesday default") + sc.add_argument("--reason", default="", help="Why (audited — required)") + sc.add_argument("--confirm", action="store_true", help="Actually write to the pipeline (else preview)") + sc.set_defaults(func=cmd_set_ccd) + + sr = sub.add_parser("skip-release", help="Suppress/cancel the release via the pipeline switch (--confirm)") + sr.add_argument("--release", required=True) + sr.add_argument("--clear", action="store_true", help="Clear the skip (re-enable the release)") + sr.add_argument("--reason", default="", help="Why (audited — required)") + sr.add_argument("--confirm", action="store_true", help="Actually write to the pipeline (else preview)") + sr.set_defaults(func=cmd_skip_release) diff --git a/release-agent/orchestrator/commands/readiness.py b/release-agent/orchestrator/commands/readiness.py new file mode 100644 index 00000000..e4a75137 --- /dev/null +++ b/release-agent/orchestrator/commands/readiness.py @@ -0,0 +1,121 @@ +"""Readiness entry-gate commands: checklist, verify, sign, decline.""" +from __future__ import annotations +import json as _json + +from orchestrator import render +from orchestrator import cli_common as C + + +def cmd_checklist(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + if getattr(args, "verify", False): + orch.gate.verify() + C.save_state(st, args.runs_root, args.release) + chk = orch.gate.checklist() + if getattr(args, "json", False): + print(_json.dumps(chk, indent=2)) + return 0 + # canonical, consistent display block (the template) — same for every engineer. + # auto-logged as scout output so the log always records what was shown. + C.emit(args.runs_root, args.release, render.readiness_table(chk, args.release), kind="readiness_checklist") + return 0 + + +def cmd_verify(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + chk = orch.gate.verify() + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_verified", + results=[{"id": it["id"], "status": it["status"]} for it in chk["auto_items"]]) + for it in chk["auto_items"]: + mark = "OK" if it["status"] == "pass" else "FAIL" + print(f" [{mark}] {it['id']}: {it['status']} — {it.get('message','')}") + return 0 + + +def cmd_sign(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + ids = None if args.all else (args.item or []) + chk = orch.gate.sign(ids) + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_signed" if chk["signed"] else "readiness_partial", + items=("all" if ids is None else ids), signed=chk["signed"]) + if chk["signed"]: + print(f"Readiness signed at {chk['signed_at']}. Entry gate cleared — you can now start Phase 0.") + else: + pending = [i["id"] for i in chk["items"] if not i["satisfied"]] + print(f"Recorded. Still pending: {', '.join(pending)}") + return 0 + + +def cmd_decline(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + chk = orch.gate.decline(args.item or []) + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_declined", items=args.item or [], blocked=chk["blocked"], driver=args.reason or None) + if chk["blocked"]: + labels = [next((i["label"] for i in chk["items"] if i["id"] == b), b) + for b in chk["blocked_items"]] + msg = ("⛔ BLOCKED — cannot start: " + ", ".join(labels) + ".\n" + " " + chk.get("blocked_message", "").strip()) + else: + msg = f"Recorded as unable: {', '.join(args.item or [])}" + C.emit(args.runs_root, args.release, msg, kind="decline_result") + return 0 + + +def cmd_record_check(args): + """Record the result of a scout-assisted auto readiness check (source: scout), + e.g. the ICM on-call lookup. The skill runs the check via its MCP tools and + calls this to store the pass/fail result in the engine.""" + st, orch = C.load_orch(args.runs_root, args.release, args.config) + res = orch.gate.record_check(args.item, args.status, args.detail or "") + if "error" in res: + print(res["error"]) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_check_recorded", item=args.item, status=args.status, driver=args.detail or None) + item = next((i for i in res["items"] if i["id"] == args.item), None) + mark = {"pass": "OK", "degraded": "WARN"}.get(args.status, "FAIL") + tail = " — entry gate cleared." if res.get("signed") else "" + C.emit(args.runs_root, args.release, + f"[{mark}] {(item or {}).get('label', args.item)}: {args.status}" + f"{(' — ' + args.detail) if args.detail else ''}{tail}", kind="record_check") + return 0 + + +def register(sub): + c = sub.add_parser("checklist", help="Show the readiness entry-gate checklist") + c.add_argument("--release", required=True) + c.add_argument("--verify", action="store_true", help="Run auto verifiers before showing") + c.add_argument("--json", action="store_true") + c.set_defaults(func=cmd_checklist) + + v = sub.add_parser("verify", help="Run the auto readiness verifiers") + v.add_argument("--release", required=True) + v.set_defaults(func=cmd_verify) + + rc = sub.add_parser("record-check", + help="Record a scout-assisted auto check result (e.g. ICM on-call)") + rc.add_argument("--release", required=True) + rc.add_argument("--item", required=True, help="Readiness item id (must be a source:scout auto item)") + rc.add_argument("--status", required=True, choices=["pass", "fail", "degraded"], + help="pass | fail (or 'degraded' for opt-out items the user proceeds without)") + rc.add_argument("--detail", default="", help="Short evidence/summary (e.g. 'not in roster')") + rc.set_defaults(func=cmd_record_check) + + sg = sub.add_parser("sign", help="Attest human readiness items (also runs auto verify)") + sg.add_argument("--release", required=True) + sg.add_argument("--all", action="store_true", help="Attest every human item") + sg.add_argument("--item", action="append", help="Attest a specific item id (repeatable)") + sg.set_defaults(func=cmd_sign) + + dc = sub.add_parser("decline", help="Declare you CANNOT satisfy an item (may block ownership)") + dc.add_argument("--release", required=True) + dc.add_argument("--item", action="append", required=True, help="Item id you cannot satisfy (repeatable)") + dc.add_argument("--reason", default="", help="Why (recorded in the event log)") + dc.set_defaults(func=cmd_decline) diff --git a/release-agent/orchestrator/commands/release.py b/release-agent/orchestrator/commands/release.py new file mode 100644 index 00000000..817645a4 --- /dev/null +++ b/release-agent/orchestrator/commands/release.py @@ -0,0 +1,295 @@ +"""Release lifecycle + manual overrides: init, list, status, next, approve, deny, +done, activate, skip, reopen, halt, resume.""" +from __future__ import annotations +import json as _json +import os + +from orchestrator.state import ReleaseState +from orchestrator.engine import Orchestrator +from orchestrator import discovery, render, schedule +from orchestrator import cli_common as C +from tools import checks + + +def cmd_init(args): + sp = C.state_path(args.runs_root, args.release) + if os.path.exists(sp) and not args.force: + print(f"Release {args.release} already exists at {sp} (use --force to recreate).") + return 1 + st = ReleaseState(release_id=args.release, dry_run=not args.live, status="not_started") + + # Release owner (the engineer running this release) — release metadata. + # Priority: explicit --owner-email (skill can pass the richer profile) then + # the signed-in az user. Never hardcoded. + st.owner_email = (getattr(args, "owner_email", None) or checks.current_az_user()) + st.owner_name = getattr(args, "owner_name", None) or None + owner_note = "" if st.owner_email else " (couldn't resolve owner — set with set-owner)" + + # CCD is canonically the 2nd Wednesday. We still READ the pipeline override, + # but we do NOT silently adopt it — if it differs, we flag a conflict for the + # user to resolve (2nd-Wed default vs the pipeline date). + src = C.ccd_source() + override, note = None, "" + if src.get("pipeline_id"): + ok, val, detail = checks.read_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["override_variable"]) + if ok: + override = val + else: + note = f" (couldn't read pipeline override: {detail})" + try: + default = schedule.default_ccd(args.release) + st.ccd = default.isoformat() + st.ccd_source = "default" + conflict = schedule.pipeline_conflict(args.release, override, st.ccd) + st.ccd_conflict = conflict.isoformat() if conflict else None + except (ValueError, IndexError): + default, conflict = None, None + note = " (release id isn't YYYY-MM — CCD not set; use set-ccd)" + st.save(sp) + + mode = "LIVE" if args.live else "dry-run" + C.elog(args.runs_root, args.release).log( + "release_started", mode=mode, forced=bool(args.force), + ccd=st.ccd, ccd_source=st.ccd_source, ccd_conflict=st.ccd_conflict, owner=st.owner_email) + owner_line = f" Owner: {st.owner_name + ' ' if st.owner_name else ''}{st.owner_email or '(unresolved)'}{owner_note}" + if st.ccd: + opens = schedule.anchor_date(default, "CCD-7").isoformat() + lines = [f"Initialized release {args.release} ({mode}).", f" state: {sp}", owner_line, + f" Code Complete Date: {st.ccd} (2nd Wednesday){note}", + f" Phase 0 (Pre-flight) opens {opens} (CCD-7). Until then nothing fires."] + if conflict: + lines.append(f" ⚠ Pipeline override is {conflict.isoformat()}, which differs from the " + f"2nd-Wednesday default. Confirm which is the real CCD before proceeding.") + print("\n".join(lines)) + else: + print(f"Initialized release {args.release} ({mode}).\n state: {sp}\n{owner_line}\n{note}") + return 0 + + +def cmd_list(args): + """List discovered releases (none/one/many). --json for the skill.""" + res = discovery.resolve(args.runs_root, getattr(args, "release", None)) + if args.json: + print(_json.dumps(res, indent=2)) + return 0 + all_ = res["all"] + if res["resolution"] == "none": + if getattr(args, "release", None): + print(f"No release '{args.release}' found. Start one with: init --release {args.release}") + else: + print("No active release on this machine. Start one with: init --release ") + return 0 + print(f"Found {len(all_)} release(s):") + for r in all_: + mark = "->" if r is res["release"] else " " + mode = "dry-run" if r["dry_run"] else "LIVE" + print(f" {mark} {r['release_id']} [{r['status']}, {mode}] updated {r['updated_at']}") + if res["resolution"] == "ambiguous": + print(f"\nMultiple releases found — assuming most recent: {res['release']['release_id']} " + f"(confirm before acting).") + return 0 + + +def cmd_status(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + if not getattr(args, "no_pipeline_check", False) and C.refresh_conflict(st): + C.save_state(st, args.runs_root, args.release) + if getattr(args, "json", False): + print(_json.dumps(orch.status_report(), indent=2)) + return 0 + C.emit(args.runs_root, args.release, render.status_view(orch.status_report()), kind="status") + return 0 + + +def cmd_next(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, args.release) # persist BEFORE any display + C.log_actions(C.elog(args.runs_root, args.release), actions) + C.emit(args.runs_root, args.release, C.advance_block(actions, orch), kind="advance") + return 0 + + +def cmd_approve(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + gate_phase, gate_step = st.current_phase, st.current_step + act = orch.approve_gate(args.comment or "") + el = C.elog(args.runs_root, args.release) + if act.kind != "idle": + el.log("gate_approved", phase=gate_phase, step=gate_step, driver=args.comment or None) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, args.release) # persist BEFORE any display + C.log_actions(el, actions) + C.emit(args.runs_root, args.release, + C.advance_block(actions, orch, lead=[f" {act.message}"]), kind="advance") + return 0 + + +def cmd_deny(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + gate_phase, gate_step = st.current_phase, st.current_step + act = orch.deny_gate(args.comment or "") + if act.kind != "idle": + C.elog(args.runs_root, args.release).log( + "gate_denied", phase=gate_phase, step=gate_step, driver=args.comment or None) + C.save_state(st, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f" {act.message}\n\n" + render.status_view(orch.status_report()), kind="deny") + return 0 + + +def cmd_done(args): + """Mark a reminder (human, non-gate) step done, then advance to the next hold.""" + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + act = orch.complete_step(getattr(args, "phase", None), getattr(args, "step", None), args.note or "") + if act.kind == "idle": + print(act.message) + return 1 + el = C.elog(args.runs_root, args.release) + el.log("reminder_done", phase=act.phase, step=act.step, driver=args.note or None) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, args.release) + C.log_actions(el, actions) + C.emit(args.runs_root, args.release, + C.advance_block(actions, orch, lead=[f" {act.message}"]), kind="advance") + return 0 + + +def cmd_skip(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + act = orch.skip_step(args.phase, args.step, args.reason or "") + if act.kind == "idle": # rejected (no reason / bad step) — nothing changed + print(act.message) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("step_skipped", phase=args.phase, step=args.step, driver=args.reason) + C.emit(args.runs_root, args.release, act.message, kind="override") + return 0 + + +def cmd_reopen(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + act = orch.reopen_step(args.phase, args.step, args.reason or "") + if act.kind == "idle": + print(act.message) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("step_reopened", phase=args.phase, step=args.step, driver=args.reason or None) + C.emit(args.runs_root, args.release, act.message, kind="override") + return 0 + + +def cmd_halt(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + act = orch.halt(args.reason or "") + if act.kind == "idle": + print(act.message) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("release_halted", driver=args.reason) + C.emit(args.runs_root, args.release, act.message, kind="override") + return 0 + + +def cmd_resume(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + act = orch.resume(args.reason or "") + if not getattr(args, "no_pipeline_check", False): + C.refresh_conflict(st) + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("release_resumed", driver=args.reason or None) + tail = "" + if st.ccd_conflict: + tail = (f"\n ⚠ Pipeline override {st.ccd_conflict} differs from CCD {st.ccd} — " + f"confirm which is correct (see status).") + C.emit(args.runs_root, args.release, (act.message + tail), kind="override") + return 0 + + +def cmd_activate(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + orch.activate_conditional(args.phase) + C.save_state(st, args.runs_root, args.release) + print(f"Activated conditional phase: {args.phase}") + return 0 + + +def register(sub): + i = sub.add_parser("init", help="Start a new release run (dry-run by default)") + i.add_argument("--release", required=True) + i.add_argument("--live", action="store_true") + i.add_argument("--force", action="store_true") + i.add_argument("--owner-email", default=None, help="Release owner email (default: signed-in az user)") + i.add_argument("--owner-name", default=None, help="Release owner display name (optional)") + i.set_defaults(func=cmd_init) + + l = sub.add_parser("list", help="Discover releases (none/one/many)") + l.add_argument("--release", required=False, default=None) + l.add_argument("--json", action="store_true") + l.set_defaults(func=cmd_list) + + s = sub.add_parser("status", help="Show the run-state brief") + s.add_argument("--release", required=True) + s.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + s.add_argument("--json", action="store_true") + s.add_argument("--no-pipeline-check", action="store_true", + help="Skip re-reading the pipeline to detect CCD drift (faster/offline).") + s.set_defaults(func=cmd_status) + + n = sub.add_parser("next", help="Advance until the next gate / completion") + n.add_argument("--release", required=True) + n.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + n.set_defaults(func=cmd_next) + + a = sub.add_parser("approve", help="Approve the current holding gate, continue") + a.add_argument("--release", required=True) + a.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + a.add_argument("--comment", default="") + a.set_defaults(func=cmd_approve) + + d = sub.add_parser("deny", help="Deny the current holding gate") + d.add_argument("--release", required=True) + d.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + d.add_argument("--comment", default="") + d.set_defaults(func=cmd_deny) + + dn = sub.add_parser("done", help="Mark a reminder (human, non-gate) step done, then advance") + dn.add_argument("--release", required=True) + dn.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + dn.add_argument("--phase", default=None, help="Defaults to the current holding step") + dn.add_argument("--step", default=None) + dn.add_argument("--note", default="", help="Optional note (audited)") + dn.set_defaults(func=cmd_done) + + # ---- manual overrides ---- + sk = sub.add_parser("skip", help="Skip a step without running it (reason REQUIRED)") + sk.add_argument("--release", required=True) + sk.add_argument("--phase", required=True) + sk.add_argument("--step", required=True) + sk.add_argument("--reason", required=True, help="Why (audit — required)") + sk.set_defaults(func=cmd_skip) + + ro = sub.add_parser("reopen", help="Reopen a done/skipped step so it runs again") + ro.add_argument("--release", required=True) + ro.add_argument("--phase", required=True) + ro.add_argument("--step", required=True) + ro.add_argument("--reason", default="", help="Why (optional)") + ro.set_defaults(func=cmd_reopen) + + ht = sub.add_parser("halt", help="Emergency hold — nothing advances until resume (reason REQUIRED)") + ht.add_argument("--release", required=True) + ht.add_argument("--reason", required=True, help="Why (audit — required)") + ht.set_defaults(func=cmd_halt) + + rs = sub.add_parser("resume", help="Clear an emergency halt") + rs.add_argument("--release", required=True) + rs.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + rs.add_argument("--no-pipeline-check", action="store_true", help="Skip CCD-drift check") + rs.add_argument("--reason", default="", help="Why (optional)") + rs.set_defaults(func=cmd_resume) + + ac = sub.add_parser("activate", help="Turn on a conditional phase (e.g. hotfix)") + ac.add_argument("--release", required=True) + ac.add_argument("--phase", required=True) + ac.set_defaults(func=cmd_activate) diff --git a/release-agent/orchestrator/discovery.py b/release-agent/orchestrator/discovery.py new file mode 100644 index 00000000..8d4d19e3 --- /dev/null +++ b/release-agent/orchestrator/discovery.py @@ -0,0 +1,72 @@ +"""Release discovery — the none / one / many logic. + +Scans the runs root for release folders and reports what's there so the +/release-agent skill can: + * 0 releases -> tell the user none is active, offer to start one + * 1 release -> use it + * many -> present the assumed one (most recently updated) + ask to confirm + +Deterministic; the skill only presents what this returns. +""" +from __future__ import annotations +import os +import json +from typing import Optional + + +def _summarize(state_file: str) -> Optional[dict]: + try: + with open(state_file, "r", encoding="utf-8") as fh: + data = json.load(fh) + except Exception: + return None + return { + "release_id": data.get("release_id"), + "status": data.get("status"), + "dry_run": data.get("dry_run", True), + "current_phase": data.get("current_phase"), + "current_step": data.get("current_step"), + "updated_at": data.get("updated_at"), + "state_file": state_file, + } + + +def list_releases(runs_root: str) -> list: + """Return summaries of all releases found, newest-updated first.""" + out = [] + if not os.path.isdir(runs_root): + return out + for name in os.listdir(runs_root): + sf = os.path.join(runs_root, name, "release-state.json") + if os.path.isfile(sf): + s = _summarize(sf) + if s: + out.append(s) + out.sort(key=lambda s: s.get("updated_at") or "", reverse=True) + return out + + +def resolve(runs_root: str, requested: Optional[str] = None) -> dict: + """Decide which release to act on. + + Returns a dict: + { "resolution": "none" | "one" | "explicit" | "ambiguous", + "release": , # the chosen/assumed release + "all": [] } # everything found + + - none : no releases exist -> caller should offer to start one + - one : exactly one exists -> use it + - explicit : caller named one and it exists -> use it + - ambiguous : several exist and none named -> 'release' is the assumed + (most recently updated); caller should confirm. + """ + all_ = list_releases(runs_root) + if requested: + match = next((r for r in all_ if r["release_id"] == requested), None) + return {"resolution": "explicit" if match else "none", + "release": match, "all": all_} + if not all_: + return {"resolution": "none", "release": None, "all": all_} + if len(all_) == 1: + return {"resolution": "one", "release": all_[0], "all": all_} + return {"resolution": "ambiguous", "release": all_[0], "all": all_} diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py new file mode 100644 index 00000000..37207370 --- /dev/null +++ b/release-agent/orchestrator/engine.py @@ -0,0 +1,695 @@ +"""Release Orchestrator — the conductor (deterministic engine, X4). + +Responsibilities (per §7.1): + 1. Load the release state machine from config/phases.yaml. + 2. Own the dispatch loop: find next step -> run its (stub) agent -> + record result -> advance, or HOLD at a gate for human approval. + 3. Persist run-state via ReleaseState (X5). + +The engine is the BRAIN: it decides what's next. The skill is only the mouth/ears. +No LLM logic here — this is fully unit-testable and dry-run-replayable. +""" +from __future__ import annotations +import os +from dataclasses import dataclass +from datetime import date +from typing import Optional + +import yaml + +from .state import ReleaseState, StepState, GateDecision, _now +from .readiness import ReadinessGate +from . import schedule +from phases import agents + + +@dataclass +class NextAction: + """What the conductor decided on this invocation — the engine's output.""" + kind: str # 'ran' | 'gate' | 'reminder' | 'scheduled' | 'complete' | 'idle' | 'readiness' | 'blocked' | 'halted' + phase: Optional[str] = None + step: Optional[str] = None + name: Optional[str] = None + message: str = "" + + +class Orchestrator: + """The conductor: owns the state machine, dispatch loop, gates, and structured + status. The readiness entry gate is delegated to ReadinessGate (self.gate); + presentation lives in render.py. This class holds no formatting logic.""" + + def __init__(self, config_path: str, state: ReleaseState, readiness_path: str = None, + as_of: date = None): + with open(config_path, "r", encoding="utf-8") as fh: + self.config = yaml.safe_load(fh) + readiness_cfg = None + if readiness_path is None: + readiness_path = os.path.join(os.path.dirname(config_path), "readiness.yaml") + if os.path.exists(readiness_path): + with open(readiness_path, "r", encoding="utf-8") as fh: + readiness_cfg = yaml.safe_load(fh) + self.state = state + self.gate = ReadinessGate(readiness_cfg, state) + # The simulated clock. Defaults to today; `--as-of` overrides it so a + # dry-run can jump to CCD-7 and prove a phase opens on schedule. + self.as_of = as_of or schedule.today() + + # ---- time anchoring (CCD-relative phase windows) ---- + def _ccd(self) -> Optional[date]: + return schedule.parse_date(self.state.ccd) + + def _phase_anchor_date(self, phase: dict) -> Optional[date]: + """The date a phase opens, or None if it has no anchor / CCD is unknown.""" + spec = phase.get("anchor") + ccd = self._ccd() + if not spec or ccd is None: + return None + return schedule.anchor_date(ccd, spec) + + def _phase_due(self, phase: dict) -> bool: + """A phase is due once the clock reaches its anchor. No anchor ⇒ always due.""" + ad = self._phase_anchor_date(phase) + return ad is None or self.as_of >= ad + + @staticmethod + def _is_reminder(step: dict) -> bool: + """A human, non-gate step is a reminder: the engine can't do it, so it + holds and tells the person to do it, then waits for them to mark it done.""" + return step.get("owner") == "human" and not step.get("gate") + + # ---- state-machine traversal ---- + def _iter_steps(self): + """Yield (phase_dict, step_dict) in definition order, skipping conditional + phases unless explicitly activated on the state.""" + for phase in self.config["phases"]: + if phase.get("conditional") and phase["id"] not in self._activated_conditionals(): + continue + for step in phase["steps"]: + yield phase, step + + def _activated_conditionals(self) -> set: + # A conditional phase (e.g. hotfix) is activated by an explicit note flag. + return {n.split("activate:")[1].strip() + for n in self.state.notes if isinstance(n, str) and n.startswith("activate:")} + + def activate_conditional(self, phase_id: str) -> None: + self.state.notes.append(f"activate:{phase_id}") + + def _first_incomplete(self): + for phase, step in self._iter_steps(): + if not self.state.is_done(phase["id"], step["id"]): + return phase, step + return None, None + + # ---- dispatch ---- + def _current_phase(self): + """The first included phase that still has incomplete steps (definition + order). Conditional phases are skipped unless activated.""" + for phase in self.config["phases"]: + if not self._phase_included(phase): + continue + if all(self.state.is_done(phase["id"], s["id"]) for s in phase["steps"]): + continue + return phase + return None + + @staticmethod + def _step_kind(step: dict) -> str: + """Classify a step: gate | scout | attest | reminder | auto.""" + if step.get("gate"): + return "gate" + if step.get("source") == "scout": + return "scout" + if step.get("attest"): + return "attest" + if step.get("owner") == "human": + return "reminder" + return "auto" + + def _deps_met(self, pid: str, step: dict) -> bool: + """True when every step this one depends_on is done (deps are within-phase).""" + for dep in step.get("depends_on", []) or []: + if not self.state.is_done(pid, dep): + return False + return True + + def step_once(self, attempted=None) -> NextAction: + """Advance exactly one step (or hold). For a sequential phase this is the + classic first-incomplete-step logic. For a parallel phase it runs one ready + step whose dependencies are met, letting independent steps progress even + when a sibling is holding. `attempted` (a set, managed by run_until_gate) + prevents re-running an auto step twice within one drain.""" + if self.state.status == "complete": + return NextAction(kind="complete", message="Release already complete.") + + # HALTED: emergency hold set by a human. Nothing advances until resume(). + if self.state.halted: + self.state.status = "halted" + return NextAction( + kind="halted", + message="Release is HALTED" + + (f": {self.state.halt_reason}" if self.state.halt_reason else "") + + ". Run resume to continue.", + ) + + # BLOCKED: an entry-gate item was declared unsatisfiable. + if self.state.blocked: + self.state.status = "blocked" + labels = self.gate.blocked_labels() + msg = (self.gate.config or {}).get("blocked_message", "").strip() + return NextAction( + kind="blocked", + message="Entry gate blocked — cannot start: " + ", ".join(labels) + ". " + msg, + ) + + # ENTRY GATE: nothing runs until the readiness checklist is signed. + if not self.state.readiness_signed: + self.state.status = "readiness_gate" + return NextAction( + kind="readiness", + message="HOLDING at the readiness entry gate. Sign the checklist before Phase 0 can start.", + ) + + phase = self._current_phase() + if phase is None: + self.state.status = "complete" + self.state.current_phase = None + self.state.current_step = None + return NextAction(kind="complete", message="All steps done — release complete.") + + self.state.current_phase = phase["id"] + + # TIME GATE: if this phase hasn't reached its anchor date yet, hold as scheduled. + if not self._phase_due(phase): + opens = self._phase_anchor_date(phase) + self.state.status = "scheduled" + days = (opens - self.as_of).days + first = next((s for s in phase["steps"] + if not self.state.is_done(phase["id"], s["id"])), None) + return NextAction( + kind="scheduled", phase=phase["id"], + step=first["id"] if first else None, + name=first["name"] if first else None, + message=f"{phase['name']} opens {opens.isoformat()} " + f"({schedule.humanize_delta(days)}). Nothing to do yet.", + ) + + if phase.get("execution") == "parallel": + return self._step_parallel(phase, attempted) + return self._step_sequential(phase) + + # ---- sequential dispatch (classic: one step at a time, stop at first hold) ---- + def _step_sequential(self, phase: dict) -> NextAction: + step = next(s for s in phase["steps"] + if not self.state.is_done(phase["id"], s["id"])) + self.state.current_step = step["id"] + + if step.get("gate") and not self._gate_approved(phase["id"], step["id"]): + self.state.status = "holding_gate" + return NextAction( + kind="gate", phase=phase["id"], step=step["id"], name=step["name"], + message=f"HOLDING at gate: {phase['name']} → {step['name']}. Awaiting human decision.", + ) + + if self._is_reminder(step): + self.state.status = "awaiting_action" + key = f"{phase['id']}.{step['id']}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + if step.get("attest"): + msg = (f"CONFIRM — attest that this is done to proceed: {step['name']}. " + f"Mark it done once you've verified it.") + else: + msg = f"ACTION NEEDED — you need to: {step['name']}. Mark it done when complete." + return NextAction(kind="reminder", phase=phase["id"], step=step["id"], + name=step["name"], message=msg) + + if step.get("source") == "scout": + self.state.status = "awaiting_action" + key = f"{phase['id']}.{step['id']}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + return NextAction( + kind="reminder", phase=phase["id"], step=step["id"], name=step["name"], + message=f"Scout-assisted check pending — {step['name']}. " + f"Scout runs this automatically when you open it.", + ) + + return self._run_auto_step(phase, step, block_holds=True) + + # ---- parallel dispatch (dependency-aware; independent steps don't block) ---- + def _step_parallel(self, phase: dict, attempted) -> NextAction: + pid = phase["id"] + + def ready(s): + return (not self.state.is_done(pid, s["id"])) and self._deps_met(pid, s) + + # 1) Run ONE ready, not-yet-attempted runnable step (auto agent, or an + # already-approved gate). Independent steps progress even if a sibling holds. + for s in phase["steps"]: + if not ready(s): + continue + kind = self._step_kind(s) + runnable = kind == "auto" or (kind == "gate" and self._gate_approved(pid, s["id"])) + if not runnable: + continue + key = f"{pid}.{s['id']}" + if attempted is not None and key in attempted: + continue + if attempted is not None: + attempted.add(key) + return self._run_auto_step(phase, s, block_holds=False) + + # 2) No more auto progress — surface the holds (all at once). + holds = [] + for s in phase["steps"]: + if not ready(s): + continue + kind = self._step_kind(s) + blocked_auto = kind == "auto" and self.state.get_step(pid, s["id"]).status == "blocked" + unapproved_gate = kind == "gate" and not self._gate_approved(pid, s["id"]) + if kind in ("scout", "attest", "reminder") or blocked_auto or unapproved_gate: + holds.append(s) + + gates = [s for s in holds if self._step_kind(s) == "gate"] + if gates: + g = gates[0] + self.state.status = "holding_gate" + self.state.current_step = g["id"] + return NextAction(kind="gate", phase=pid, step=g["id"], name=g["name"], + message=f"HOLDING at gate: {phase['name']} → {g['name']}. Awaiting human decision.") + + non_gate = [s for s in holds if self._step_kind(s) != "gate"] + for s in non_gate: + key = f"{pid}.{s['id']}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + if non_gate: + self.state.status = "awaiting_action" + self.state.current_step = non_gate[0]["id"] + names = "; ".join(s["name"] for s in non_gate) + return NextAction(kind="reminder", phase=pid, step=non_gate[0]["id"], + name=non_gate[0]["name"], + message=f"{len(non_gate)} item(s) need you: {names}") + + # Not complete, but nothing is ready — remaining steps wait on unmet deps. + self.state.status = "awaiting_action" + return NextAction(kind="reminder", phase=pid, + message="Waiting on prerequisite steps to complete.") + + def _run_auto_step(self, phase: dict, step: dict, block_holds: bool) -> NextAction: + """Run an agent step. On success → done. On failure: in sequential mode + (block_holds=True) HOLD as action-needed and break; in parallel mode + (block_holds=False) mark it blocked + register it, but return 'ran' so the + drain continues with independent steps.""" + pid = phase["id"] + agent_id = step.get("agent", "stub") + runner = agents.get_runner(agent_id) + result = runner(pid, step, self.state.dry_run, self.state) + key = f"{pid}.{step['id']}" + if not result.ok: + self.state.set_step(pid, step["id"], + StepState(status="blocked", note=result.action, by=result.by)) + if key not in self.state.pending_human: + self.state.pending_human.append(key) + if block_holds: + self.state.status = "awaiting_action" + return NextAction(kind="reminder", phase=pid, step=step["id"], + name=step["name"], + message=f"ACTION NEEDED — {step['name']}: {result.action}") + return NextAction(kind="ran", phase=pid, step=step["id"], name=step["name"], + message=f"BLOCKED — {step['name']}: {result.action}") + self.state.set_step(pid, step["id"], + StepState(status="done", completed_at=_now(), + note=result.action, by=result.by)) + if result.by == "human": + self.state.pending_human = [p for p in self.state.pending_human if p != key] + self.state.status = "running" + return NextAction(kind="ran", phase=pid, step=step["id"], + name=step["name"], message=result.action) + + def run_until_gate(self, max_steps: int = 500) -> list: + """Drive the loop until a gate hold, completion, or step cap. + Returns the list of NextAction taken. `attempted` prevents re-running an + auto step twice within this drain (so a re-blocking step can't loop).""" + actions = [] + attempted = set() + for _ in range(max_steps): + act = self.step_once(attempted) + actions.append(act) + if act.kind in ("gate", "reminder", "scheduled", "complete", "readiness", "blocked", "halted"): + break + return actions + + # ---- manual overrides (human-driven transitions, §7.1 constraint #5) ---- + def _find_step(self, phase_id: str, step_id: str): + phase = next((p for p in self.config["phases"] if p["id"] == phase_id), None) + if phase and any(s["id"] == step_id for s in phase["steps"]): + return phase + return None + + def skip_step(self, phase_id: str, step_id: str, reason: str) -> NextAction: + """Mark a step skipped (counts as done for progression) without running it. + A reason is REQUIRED (audit). For 'doesn't apply' or 'done manually outside the tool'.""" + if not (reason and reason.strip()): + return NextAction(kind="idle", message="A reason is required to skip a step.") + if not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + self.state.set_step(phase_id, step_id, + StepState(status="skipped", completed_at=_now(), + note=f"Skipped: {reason.strip()}", by="human")) + if self.state.status == "holding_gate" and self.state.current_step == step_id: + self.state.status = "running" + return NextAction(kind="ran", phase=phase_id, step=step_id, + message=f"Skipped {phase_id}/{step_id} — {reason.strip()}") + + def complete_step(self, phase_id: str = None, step_id: str = None, note: str = "") -> NextAction: + """Mark a reminder (human, non-gate) step done. Defaults to the step the + conductor is currently holding on. This is how a person clears an + 'ACTION NEEDED' hold once they've actually done the task.""" + phase_id = phase_id or self.state.current_phase + step_id = step_id or self.state.current_step + if not (phase_id and step_id) or not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + self.state.set_step(phase_id, step_id, + StepState(status="done", completed_at=_now(), + note=(note.strip() or "Marked done"), by="human")) + key = f"{phase_id}.{step_id}" + self.state.pending_human = [p for p in self.state.pending_human + if p != key and not p.startswith(key + " ")] + if self.state.status == "awaiting_action": + self.state.status = "running" + tail = f" — {note.strip()}" if note and note.strip() else "" + return NextAction(kind="ran", phase=phase_id, step=step_id, + message=f"Done: {phase_id}/{step_id}{tail}") + + def record_scout_step(self, phase_id: str, step_id: str, status: str, + detail: str = "") -> NextAction: + """Record the outcome of a scout-assisted step (one the skill ran via MCP/ + browser, e.g. the CCOA lockdown check). + * status == "pass" -> mark the step done and let the flow continue. + * status == "attention" -> keep it held (needs the owner) with the detail + (e.g. a Production CCOA lockdown overlaps — the owner must shift CCD).""" + if not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + if status == "pass": + return self.complete_step(phase_id, step_id, detail) + # attention: leave the step outstanding, flagged for the owner. + self.state.set_step(phase_id, step_id, + StepState(status="blocked", note=detail, by="scout")) + self.state.status = "awaiting_action" + key = f"{phase_id}.{step_id}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + return NextAction(kind="reminder", phase=phase_id, step=step_id, + message=f"Needs your attention — {detail}") + + def reopen_step(self, phase_id: str, step_id: str, reason: str = "") -> NextAction: + """Undo a done/skipped step so the conductor runs it again. Reason optional.""" + if not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + self.state.steps.pop(self.state.key(phase_id, step_id), None) # remove -> pending + # drop any prior gate approval for this step so a gate re-holds + self.state.gate_decisions = [g for g in self.state.gate_decisions + if g.get("step") != f"{phase_id}.{step_id}"] + if self.state.status == "complete": + self.state.status = "running" + note = f" — {reason.strip()}" if reason and reason.strip() else "" + return NextAction(kind="ran", phase=phase_id, step=step_id, + message=f"Reopened {phase_id}/{step_id}{note}") + + def halt(self, reason: str) -> NextAction: + """Emergency hold — nothing advances until resume(). Reason REQUIRED (audit).""" + if not (reason and reason.strip()): + return NextAction(kind="idle", message="A reason is required to halt the release.") + self.state.halted = True + self.state.halt_reason = reason.strip() + self.state.status = "halted" + return NextAction(kind="halted", message=f"Release HALTED — {reason.strip()}") + + def resume(self, reason: str = "") -> NextAction: + """Clear an emergency halt. Reason optional.""" + if not self.state.halted: + return NextAction(kind="idle", message="Release is not halted.") + self.state.halted = False + self.state.halt_reason = None + self.state.status = "running" + note = f" — {reason.strip()}" if reason and reason.strip() else "" + return NextAction(kind="idle", message=f"Release resumed{note}.") + + # ---- gates ---- + def _gate_approved(self, phase: str, step: str) -> bool: + for gd in self.state.gate_decisions: + if gd.get("step") == f"{phase}.{step}" and gd.get("decision") == "approved": + return True + return False + + def approve_gate(self, comment: str = "") -> NextAction: + """Record approval for the current holding gate and continue.""" + phase = self.state.current_phase + step = self.state.current_step + if self.state.status != "holding_gate" or not phase: + return NextAction(kind="idle", message="No gate is currently holding.") + self.state.gate_decisions.append( + asdict_gate(GateDecision(step=f"{phase}.{step}", decision="approved", + at=_now(), comment=comment))) + # mark the gate step done and advance + self.state.set_step(phase, step, + StepState(status="done", completed_at=_now(), + note=f"Gate approved. {comment}".strip(), by="human")) + self.state.status = "running" + return NextAction(kind="ran", phase=phase, step=step, + message=f"Gate approved: {phase} → {step}. {comment}".strip()) + + def deny_gate(self, comment: str = "") -> NextAction: + phase = self.state.current_phase + step = self.state.current_step + if self.state.status != "holding_gate" or not phase: + return NextAction(kind="idle", message="No gate is currently holding.") + self.state.gate_decisions.append( + asdict_gate(GateDecision(step=f"{phase}.{step}", decision="denied", + at=_now(), comment=comment))) + self.state.pending_human.append(f"{phase}.{step} (denied: {comment})") + self.state.status = "blocked" + return NextAction(kind="gate", phase=phase, step=step, + message=f"Gate DENIED: {phase} → {step}. Release blocked. {comment}".strip()) + + # ---- reporting ---- + def _phase_included(self, phase: dict) -> bool: + return (not phase.get("conditional")) or phase["id"] in self._activated_conditionals() + + def _active_phase_report(self) -> Optional[dict]: + """The first incomplete included phase, with its outstanding steps and + whether its time-window is open (due). This is what the daily phase + notification reports on — independent of state.current_phase (which is + only set once the release has been advanced).""" + for phase in self.config["phases"]: + if not self._phase_included(phase): + continue + steps = phase["steps"] + done = sum(1 for s in steps if self.state.is_done(phase["id"], s["id"])) + if done == len(steps): + continue # phase complete — look at the next one + outstanding = [ + {"id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), "owner": s.get("owner", "agent")} + for s in steps if not self.state.is_done(phase["id"], s["id"]) + ] + completed = [s["name"] for s in steps + if self.state.is_done(phase["id"], s["id"])] + cur = self.state.current_step + steps_view = [] + for s in steps: + sid = s["id"] + s_done = self.state.is_done(phase["id"], sid) + s_blocked = self.state.get_step(phase["id"], sid).status == "blocked" + is_gate = bool(s.get("gate")) + is_rem = self._is_reminder(s) + is_scout = s.get("source") == "scout" + is_attest = bool(s.get("attest")) + if s_done: + status = "done" + elif s_blocked: + status = "blocked" + elif is_gate: + status = "approval" + elif is_attest: + status = "confirm" + elif is_rem or is_scout: + status = "action" + else: + status = "auto" + needs = bool((is_gate or is_rem or is_scout or is_attest or s_blocked) and not s_done) + steps_view.append({ + "id": sid, "name": s["name"], "status": status, + "needs_owner": needs, + "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_scout or is_attest or s_blocked)), + }) + opens = self._phase_anchor_date(phase) + return { + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase"), + "done": done, "total": len(steps), + "due": self._phase_due(phase), "started": done > 0, + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + "outstanding": outstanding, + "completed": completed, + "steps": steps_view, + } + return None + + def _phase_map(self): + """Build the phase overview + running totals. Returns + (phases, total, done, current_phase_name, current_phase_obj, current_step_name).""" + phases = [] + total = done = 0 + current_phase_name = current_step_name = None + current_phase_obj = None + for idx, phase in enumerate(self.config["phases"]): + if not self._phase_included(phase): + continue + p_total = len(phase["steps"]) + p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) + total += p_total + done += p_done + is_current = self.state.current_phase == phase["id"] + due = self._phase_due(phase) + opens = self._phase_anchor_date(phase) + if p_total and p_done == p_total: + state = "done" + elif not due and p_done == 0: + state = "scheduled" + elif is_current or p_done > 0: + state = "current" + else: + state = "pending" + if is_current: + current_phase_name = phase["name"] + current_phase_obj = phase + phases.append({ + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase", idx), + "done": p_done, "total": p_total, "state": state, + "current": is_current, + "anchor": phase.get("anchor"), + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + }) + for s in phase["steps"]: + if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: + current_step_name = s["name"] + return phases, total, done, current_phase_name, current_phase_obj, current_step_name + + def _current_steps(self, current_phase_obj) -> list: + """The current phase's steps, each tagged with a display state.""" + if not current_phase_obj: + return [] + phase_due = self._phase_due(current_phase_obj) + out = [] + for s in current_phase_obj["steps"]: + rec = self.state.steps.get(self.state.key(current_phase_obj["id"], s["id"]), {}) or {} + if rec.get("status") == "skipped": + s_state = "skipped" + elif self.state.is_done(current_phase_obj["id"], s["id"]): + s_state = "done" + elif s["id"] == self.state.current_step and self.state.status == "holding_gate": + s_state = "gate" + elif s["id"] == self.state.current_step and self.state.status == "awaiting_action": + s_state = "reminder" + elif not phase_due: + s_state = "scheduled" + else: + s_state = "pending" + out.append({ + "id": s["id"], "name": s["name"], + "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), + "owner": s.get("owner", "agent"), + "state": s_state, + }) + return out + + def _hold_view(self, phase_name, step_name) -> dict: + """Detail of the current hold (gate or action-needed) — same shape for both.""" + return { + "phase": self.state.current_phase, + "phase_name": phase_name, + "step": self.state.current_step, + "step_name": step_name, + } + + def _scheduled_view(self) -> Optional[dict]: + """The phase we're waiting on the clock for — derived from the first + incomplete phase's due-ness, so `status` shows it even before `next`.""" + first_incomplete = next( + (p for p in self.config["phases"] + if self._phase_included(p) + and not all(self.state.is_done(p["id"], s["id"]) for s in p["steps"])), + None) + if (first_incomplete is None or self._phase_due(first_incomplete) + or self.state.status in ("complete", "halted", "blocked")): + return None + opens = self._phase_anchor_date(first_incomplete) + return { + "phase": first_incomplete["id"], + "phase_name": first_incomplete["name"], + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + } + + def status_report(self) -> dict: + """Structured status — presentation layer (render.py) turns this into a view. + Deterministic; no formatting baked in. Assembled from focused builders: + phase map, current-phase steps, current hold, scheduled window, active phase.""" + (phases, total, done, current_phase_name, + current_phase_obj, current_step_name) = self._phase_map() + current_steps = self._current_steps(current_phase_obj) + + gate = action = None + if self.state.status == "holding_gate" and self.state.current_phase: + gate = self._hold_view(current_phase_name, current_step_name) + elif self.state.status == "awaiting_action" and self.state.current_phase: + action = self._hold_view(current_phase_name, current_step_name) + scheduled = self._scheduled_view() + + chk = self.gate.checklist() + return { + "release_id": self.state.release_id, + "status": self.state.status, + "dry_run": self.state.dry_run, + "owner_email": self.state.owner_email, + "owner_name": self.state.owner_name, + "ccd": self.state.ccd, + "ccd_source": self.state.ccd_source, + "ccd_conflict": self.state.ccd_conflict, + "as_of": self.as_of.isoformat(), + "skip_release": self.state.skip_release, + "readiness_signed": self.state.readiness_signed, + "readiness_pending": [i["id"] for i in chk["items"] if not i["satisfied"]], + "blocked": self.state.blocked, + "blocked_items": list(self.state.blocked_items), + "blocked_message": chk.get("blocked_message", ""), + "halted": self.state.halted, + "halt_reason": self.state.halt_reason, + "done": done, "total": total, + "percent": round(100 * done / total) if total else 0, + "phases": phases, + "current_phase": self.state.current_phase, + "current_phase_name": current_phase_name, + "current_step": self.state.current_step, + "current_step_name": current_step_name, + "current_steps": current_steps, + "gate": gate, + "action": action, + "scheduled": scheduled, + "active_phase": self._active_phase_report(), + "pending_human": list(self.state.pending_human), + "gate_decisions": len(self.state.gate_decisions), + "updated_at": self.state.updated_at, + } + + +def asdict_gate(gd: GateDecision) -> dict: + return {"step": gd.step, "decision": gd.decision, "at": gd.at, + "by": gd.by, "comment": gd.comment} diff --git a/release-agent/orchestrator/eventlog.py b/release-agent/orchestrator/eventlog.py new file mode 100644 index 00000000..7df01951 --- /dev/null +++ b/release-agent/orchestrator/eventlog.py @@ -0,0 +1,108 @@ +"""Event log — the per-release record we analyze to debug and improve. + +Scope: PER-RELEASE ONLY. Each release keeps its own complete log at + //events.jsonl +There is no machine-wide aggregate — a release is a self-contained unit and its +log travels with it. + +What it must capture to be useful for debugging: not just the engine commands, +but the actual INTERACTION — + * what Scout presented to the engineer (prompts, the rendered checklist, options) + * what the engineer chose / typed (their input) + * the engine's own events (steps, gate holds, decisions with drivers) + +Each line is one JSON object: + { ts, release_id, actor, source, event, ... } + source: "engine" (deterministic engine action) | "scout" (agent output) | "user" (engineer input) + +Logging is best-effort and MUST never break or alter the interaction. +""" +from __future__ import annotations +import json +import os +import getpass +from datetime import datetime, timezone + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _actor() -> str: + try: + return getpass.getuser() + except Exception: + return os.getenv("USERNAME") or os.getenv("USER") or "unknown" + + +class EventLog: + def __init__(self, runs_root: str, release_id: str): + self.runs_root = runs_root + self.release_id = release_id + self.path = os.path.join(runs_root, release_id, "events.jsonl") + self.actor = _actor() + + def log(self, event: str, source: str = "engine", **fields) -> dict: + rec = {"ts": _now(), "release_id": self.release_id, "actor": self.actor, + "source": source, "event": event} + rec.update({k: v for k, v in fields.items() if v is not None}) + try: + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(rec, ensure_ascii=False) + "\n") + except Exception: + pass # logging must never break the flow + return rec + + # convenience wrappers for the interaction layer (called by the skill via CLI) + def scout_said(self, text: str, kind: str = "message", options=None) -> dict: + return self.log("scout_output", source="scout", kind=kind, text=text, options=options) + + def user_said(self, text: str, kind: str = "input", choice=None) -> dict: + return self.log("user_input", source="user", kind=kind, text=text, choice=choice) + + def read(self, limit: int = None) -> list: + return _read_jsonl(self.path, limit) + + +def _read_jsonl(path: str, limit: int = None) -> list: + if not os.path.isfile(path): + return [] + out = [] + with open(path, "r", encoding="utf-8") as fh: + for ln in fh: + ln = ln.strip() + if not ln: + continue + try: + out.append(json.loads(ln)) + except Exception: + continue + return out[-limit:] if limit else out + + +def summarize(events: list) -> dict: + """Roll up a single release's events for quick debugging.""" + by_event, by_source = {}, {} + gate_decisions, declines, interactions = [], [], 0 + for e in events: + by_event[e.get("event")] = by_event.get(e.get("event"), 0) + 1 + by_source[e.get("source")] = by_source.get(e.get("source"), 0) + 1 + if e.get("source") in ("scout", "user"): + interactions += 1 + if e.get("event") in ("gate_approved", "gate_denied"): + gate_decisions.append({"phase": e.get("phase"), "step": e.get("step"), + "decision": "approved" if e["event"] == "gate_approved" else "denied", + "driver": e.get("driver"), "actor": e.get("actor"), "ts": e.get("ts")}) + if e.get("event") == "readiness_declined": + declines.append({"items": e.get("items"), "owner_blocked": e.get("owner_blocked"), + "driver": e.get("driver"), "ts": e.get("ts")}) + return { + "release": events[0]["release_id"] if events else None, + "total_events": len(events), + "by_source": by_source, + "by_event": by_event, + "interactions_logged": interactions, + "gate_decisions": gate_decisions, + "declines": declines, + } diff --git a/release-agent/orchestrator/infra.py b/release-agent/orchestrator/infra.py new file mode 100644 index 00000000..3da3ba1a --- /dev/null +++ b/release-agent/orchestrator/infra.py @@ -0,0 +1,177 @@ +"""Infrastructure preflight — verify (and auto-provision) everything the skill +needs on a machine BEFORE the tool-level requirements: CLIs, the launchers that +back MCP servers, and the MCP servers themselves registered into Scout's config. + +Design: + * CLI/host/python deps ("requirements") → shell-checked (same as before). + * MCP servers ("mcp_servers") → live in Scout's own config file + (~/.scout/m-mcp-servers.json), which loads at startup. We can't "install" them, + but we CAN register a missing one into that file (backing it up first) so it + loads on the next Scout restart. Each entry names its `provider` (a shell check + that the launcher exists) so we never register a server whose launcher is absent. + +This module is pure-Python and import-light so bootstrap.ps1 can call it via the +CLI (`python -m orchestrator.cli infra`). It only touches Scout's MCP config when +asked to register, and always backs it up. +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from datetime import datetime + +import yaml + + +def scout_mcp_config_path() -> str: + return os.path.join(os.path.expanduser("~"), ".scout", "m-mcp-servers.json") + + +def expand(s: str) -> str: + """Expand %VARS% / $VARS and ~ in a path string.""" + return os.path.expanduser(os.path.expandvars(s or "")) + + +def load_requirements(path: str) -> dict: + with open(path, "r", encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + + +def _shell_ok(cmd: str, timeout: int = 30) -> bool: + """True if the shell command exits 0. Used for CLI + provider checks.""" + if not cmd: + return False + try: + r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) + return r.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def check_requirements(req: dict) -> list: + """Return [{id,name,ok,install}] for each shell-checkable requirement.""" + out = [] + for r in req.get("requirements", []): + out.append({ + "id": r.get("id", ""), "name": r.get("name", r.get("id", "?")), + "ok": _shell_ok(r.get("check", "")), "install": r.get("install", ""), + }) + return out + + +def _load_scout_config(path: str) -> dict: + if not os.path.exists(path): + return {"servers": {}} + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + data.setdefault("servers", {}) + return data + + +def _backup(path: str) -> str: + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + dst = f"{path}.bak-{stamp}" + shutil.copy2(path, dst) + return dst + + +def ensure_mcp_servers(req: dict, register: bool) -> list: + """Check each required MCP server against Scout's config. When `register` is + True, add any that are missing (whose provider launcher exists), backing up + the config once before the first write. + + Returns [{id,name,scout_key,status,detail}] where status is one of: + present | registered | would_register | provider_missing | launcher_missing + """ + servers = req.get("mcp_servers", []) + if not servers: + return [] + cfg_path = scout_mcp_config_path() + cfg = _load_scout_config(cfg_path) + existing = cfg.get("servers", {}) + results, dirty, backed_up = [], False, None + + for m in servers: + key = m.get("scout_key") or m.get("id") + name = m.get("name", key) + rec = {"id": m.get("id"), "name": name, "scout_key": key} + if key in existing: + results.append({**rec, "status": "present", "detail": "already in Scout config"}) + continue + # Not registered. Is its launcher present? + provider_ok = _shell_ok(m.get("provider", "")) + cmd = expand(m.get("command", "")) + launcher_ok = bool(cmd) and os.path.exists(cmd) + if not provider_ok and not launcher_ok: + results.append({**rec, "status": "provider_missing", + "detail": m.get("note", "provider/launcher not found")}) + continue + if not launcher_ok: + results.append({**rec, "status": "launcher_missing", + "detail": f"launcher not found at {cmd or '(unset)'}"}) + continue + if not register: + results.append({**rec, "status": "would_register", + "detail": "run bootstrap (or infra --register) to add it"}) + continue + # Build args, expanding any dynamic directive (e.g. Kusto known-services + # from a data list) so multi-cluster config stays pure data. + args = list(m.get("args", [])) + ks_from = m.get("known_services_from") + if ks_from: + clusters = req.get(ks_from, []) or [] + known = [{"service_uri": c.get("service_uri"), + "default_database": c.get("default_database"), + "description": c.get("description", "")} + for c in clusters if c.get("service_uri")] + if known: + args += ["--known-services", json.dumps(known)] + # Register it into the config (backup once). + if not backed_up and os.path.exists(cfg_path): + backed_up = _backup(cfg_path) + existing[key] = { + "builtin": False, + "config": {"name": name.split(" MCP")[0].strip() or key, + "type": "command", "command": cmd, "args": args}, + "tools": [], + } + dirty = True + results.append({**rec, "status": "registered", + "detail": "added to Scout config — RESTART Scout to load"}) + + if dirty: + os.makedirs(os.path.dirname(cfg_path), exist_ok=True) + tmp = cfg_path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(cfg, fh, indent=2, ensure_ascii=False) + os.replace(tmp, cfg_path) + return results + + +def run(req_path: str, register: bool = True) -> dict: + """Full infra preflight. Returns a structured report. + + If Scout itself isn't present (~/.scout missing), MCP registration is skipped + (there's no config to write) and the report flags scout_missing — install Scout + first, then re-run. + """ + req = load_requirements(req_path) + reqs = check_requirements(req) + scout_present = os.path.isdir(os.path.join(os.path.expanduser("~"), ".scout")) + if scout_present: + mcps = ensure_mcp_servers(req, register) + else: + # can't register into a non-existent Scout config — report as pending + mcps = [{"id": m.get("id"), "name": m.get("name", m.get("id")), + "scout_key": m.get("scout_key") or m.get("id"), + "status": "scout_missing", "detail": "install Scout first"} + for m in req.get("mcp_servers", [])] + restart_needed = any(m["status"] == "registered" for m in mcps) + ok = (scout_present and all(r["ok"] for r in reqs) + and all(m["status"] in ("present", "registered") for m in mcps)) + return {"requirements": reqs, "mcp_servers": mcps, + "scout_present": scout_present, + "ok": ok, "restart_needed": restart_needed, + "scout_mcp_config": scout_mcp_config_path()} diff --git a/release-agent/orchestrator/phase_config.py b/release-agent/orchestrator/phase_config.py new file mode 100644 index 00000000..6bc73031 --- /dev/null +++ b/release-agent/orchestrator/phase_config.py @@ -0,0 +1,38 @@ +"""Per-phase config loader — one convention: config/.yaml. + +Each phase's config lives beside the phase and mirrors its code +(phases/agents/.py ↔ config/.yaml), so adding a phase is a +predictable recipe with no hard-coded filenames scattered across modules. + +Cross-cutting config (requirements.yaml, schedule.yaml, readiness.yaml) is +release-wide, NOT per-phase, and is loaded elsewhere — it does not go through here. + +Import-light (os + yaml only) so both the phase agents (phases/) and the CLI +command modules (orchestrator/commands/) can use it without a cycle. +""" +from __future__ import annotations + +import os + +import yaml + +_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def phase_config_path(phase_id: str) -> str: + """Absolute path to a phase's config file: config/.yaml.""" + return os.path.join(_ROOT, "config", f"{phase_id}.yaml") + + +def load_phase_config(phase_id: str, section: str | None = None) -> dict: + """Load config/.yaml. With `section`, return just that top-level key + (e.g. load_phase_config('preflight', 'cg')); without it, the whole document. + Missing file or key → empty dict (agents fall back to their own defaults).""" + try: + with open(phase_config_path(phase_id), "r", encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + except OSError: + data = {} + if section is not None: + return data.get(section, {}) or {} + return data diff --git a/release-agent/orchestrator/readiness.py b/release-agent/orchestrator/readiness.py new file mode 100644 index 00000000..2d35ccde --- /dev/null +++ b/release-agent/orchestrator/readiness.py @@ -0,0 +1,201 @@ +"""Release readiness entry gate (logic only — no presentation). + +A self-contained subsystem: given the readiness config (data) + the release +run-state, it computes the checklist, runs auto verifiers, records attestations +and declines, and decides whether the gate is signed/blocked. + +Returns structured data only. Rendering lives in render.py so a different +interface (web UI, other frontend) can present the same data its own way. + +Model: every item is equally required. The only distinction is WHO resolves it: + auto -> Scout verifies it programmatically (pass/fail) + attest -> the engineer confirms it +If any item is unsatisfied the gate is not signed. If the engineer declares they +cannot satisfy an item (decline), the gate is blocked until resolved / handed off. +""" +from __future__ import annotations + +from . import schedule +from .state import ReleaseState, _now + + +class ReadinessGate: + def __init__(self, config: dict, state: ReleaseState): + # config is the parsed readiness.yaml (or None if not configured) + self.config = config + self.state = state + + def _window(self, it: dict): + """Compute a CCD-relative window {start,end} for an item that declares + window_start_anchor / window_end_anchor, using the release CCD. Returns + None when the item has no window or CCD isn't set yet.""" + sa, ea = it.get("window_start_anchor"), it.get("window_end_anchor") + ccd = schedule.parse_date(self.state.ccd) + if not (sa and ea and ccd): + return None + try: + return {"start": schedule.anchor_date(ccd, sa).isoformat(), + "end": schedule.anchor_date(ccd, ea).isoformat()} + except ValueError: + return None + + # ---- queries ---- + def checklist(self) -> dict: + """The entry checklist as structured data (no formatting).""" + if not self.config: + return {"items": [], "signed": True, "title": "", "instructions": ""} + items = [] + for it in self.config.get("items", []): + rec = self.state.readiness_items.get(it["id"], {}) or {} + links = [] + for ln in it.get("links", []): + if isinstance(ln, dict): + links.append({"name": ln.get("name", ln.get("url")), "url": ln.get("url")}) + else: + links.append({"name": ln, "url": ln}) + items.append({ + "id": it["id"], "text": it["text"], + "label": it.get("label", it["id"]), + "detail": it.get("detail"), + "links": links, + "verify": it.get("verify", "attest"), # auto | attest (who resolves it) + "source": it.get("source"), # None (python) | "scout" (skill runs it via MCP) + "verifier": it.get("verifier"), + "team_id": it.get("team_id"), # for scout-assisted checks (e.g. on-call team) + "team_name": it.get("team_name"), + "cluster_uri": it.get("cluster_uri"), # for scout-assisted Kusto checks + "database": it.get("database"), + "required_servers": it.get("required_servers"), # for scout-assisted silent-perms check + "opt_out": it.get("opt_out", False), # soft item: may be waived ("degraded") and still satisfy + "window": self._window(it), # {start,end} for windowed attest items + "status": rec.get("status", "pending"), # pending | pass | fail | attested | unable + "message": rec.get("message"), + "checks": rec.get("checks", []), # per-check results for auto items + "satisfied": self._item_satisfied(it), + }) + return { + "title": self.config.get("title", "Release readiness"), + "instructions": self.config.get("instructions", ""), + "blocked_message": self.config.get("blocked_message", ""), + "items": items, + "auto_items": [i for i in items if i["verify"] == "auto"], + "attest_items": [i for i in items if i["verify"] == "attest"], + "signed": self.state.readiness_signed, + "signed_at": self.state.readiness_signed_at, + "blocked": self.state.blocked, + "blocked_items": list(self.state.blocked_items), + "all_satisfied": all(i["satisfied"] for i in items) if items else True, + } + + @property + def signed(self) -> bool: + return self.state.readiness_signed + + @property + def blocked(self) -> bool: + return self.state.blocked + + # ---- mutations ---- + def verify(self) -> dict: + """Run the AUTO verifiers. Each returns pass or fail — no half-measures. + Items with source: scout are skipped here — the skill runs those via its + MCP tools and records the result with record_check().""" + from phases.readiness_verifiers import get_verifier + if not self.config: + return self.checklist() + for it in self.config.get("items", []): + if it.get("verify") != "auto": + continue + if it.get("source") == "scout": + continue # skill-run (MCP) — not executable here + vf = get_verifier(it.get("verifier")) + if vf is None: + self.state.readiness_items[it["id"]] = {"status": "fail", + "message": f"no verifier '{it.get('verifier')}' registered", "at": _now()} + continue + res = vf(it, self.state.dry_run) + rec = {"status": res.status, "message": res.message, "at": _now()} + if getattr(res, "details", None): + rec["checks"] = res.details + self.state.readiness_items[it["id"]] = rec + self._refresh_signed() + return self.checklist() + + def record_check(self, item_id: str, status: str, message: str = "") -> dict: + """Record the result of a SCOUT-ASSISTED auto check (source: scout), which + the skill performs via its MCP tools (e.g. the ICM on-call lookup). Only + valid for auto items marked source: scout — a Python-verified auto item + (e.g. build_access) cannot be hand-recorded, and attest items use sign(). + + Status is normally 'pass' | 'fail'. Items marked `opt_out: true` (soft items + the user may proceed WITHOUT — e.g. silent_perms) also accept 'degraded', + which SATISFIES the gate while recording that they chose to proceed without + the capability (the downside is captured in `message`).""" + by_id = {it["id"]: it for it in (self.config or {}).get("items", [])} + it = by_id.get(item_id) + if it is None: + return {"error": f"no such readiness item '{item_id}'"} + if not (it.get("verify") == "auto" and it.get("source") == "scout"): + return {"error": f"'{item_id}' is not a scout-assisted auto item (cannot record a result for it)"} + valid = ("pass", "fail") + (("degraded",) if it.get("opt_out") else ()) + if status not in valid: + return {"error": f"status must be one of: {', '.join(valid)}"} + self.state.readiness_items[item_id] = { + "status": status, "message": message, "at": _now(), "source": "scout"} + self._refresh_signed() + return self.checklist() + + def sign(self, item_ids=None) -> dict: + """Attest human (attest) items and run auto verifiers. item_ids=None + attests every attest item. Auto items are only set by verification — + they cannot be hand-waved through. Signs when all items are satisfied.""" + if not self.config: + self.state.readiness_signed = True + self.state.readiness_signed_at = _now() + return self.checklist() + self.verify() # auto items (pass/fail) + all_items = self.config.get("items", []) + attest_ids = [it["id"] for it in all_items if it.get("verify", "attest") == "attest"] + targets = attest_ids if item_ids is None else [i for i in item_ids if i in attest_ids] + for iid in targets: + self.state.readiness_items[iid] = {"status": "attested", "at": _now()} + self._refresh_signed() + return self.checklist() + + def decline(self, item_ids: list) -> dict: + """The engineer declares they CANNOT satisfy one or more items. Every item + is required, so any declined item blocks the gate until resolved / handed off.""" + if not self.config: + return self.checklist() + by_id = {it["id"]: it for it in self.config.get("items", [])} + for iid in item_ids or []: + if iid not in by_id: + continue + self.state.readiness_items[iid] = {"status": "unable", "at": _now()} + self.state.blocked = True + if iid not in self.state.blocked_items: + self.state.blocked_items.append(iid) + self.state.readiness_signed = False + return self.checklist() + + def blocked_labels(self) -> list: + """Human labels for the currently-blocked item ids.""" + by_id = {it["id"]: it.get("label", it["id"]) for it in (self.config or {}).get("items", [])} + return [by_id.get(b, b) for b in self.state.blocked_items] + + # ---- internals ---- + def _item_satisfied(self, item: dict) -> bool: + status = (self.state.readiness_items.get(item["id"], {}) or {}).get("status") + if item.get("verify", "attest") == "auto": + if item.get("opt_out"): + # soft item: fully verified (pass) OR user chose to proceed (degraded) + return status in ("pass", "degraded") + return status == "pass" # auto is fully verified: pass or nothing + return status == "attested" # attest is human-confirmed + + def _refresh_signed(self) -> None: + items = self.config.get("items", []) if self.config else [] + if items and all(self._item_satisfied(it) for it in items): + if not self.state.readiness_signed: + self.state.readiness_signed = True + self.state.readiness_signed_at = _now() diff --git a/release-agent/orchestrator/registry.py b/release-agent/orchestrator/registry.py new file mode 100644 index 00000000..231e6ac5 --- /dev/null +++ b/release-agent/orchestrator/registry.py @@ -0,0 +1,77 @@ +"""Automation registry — tracks the Scout automations the orchestrator provisions +for a release, so they can be cleanly torn down at release close. + +The engine/CLI never call Scout's automation API (creating/deleting automations is +the skill's job via m_create_automation / m_delete_automation). This module only +RECORDS which automations exist, tagged by release + scope, so the skill knows +exactly what to remove at the end — nothing gets orphaned. + +Two scopes: + * shared — machine-wide, reused across releases (e.g. "Release push reminders"). + NOT torn down per release. + * release — provisioned for one release; removed when that release closes. + +Stored machine-wide at /_automations.json (gitignored runtime state). +""" +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class AutomationRegistry: + def __init__(self, runs_root: str): + self.runs_root = runs_root + self.path = os.path.join(runs_root, "_automations.json") + + def _load(self) -> list: + try: + with open(self.path, "r", encoding="utf-8") as fh: + data = json.load(fh) + return data if isinstance(data, list) else [] + except (OSError, ValueError): + return [] + + def _save(self, entries: list) -> None: + os.makedirs(self.runs_root, exist_ok=True) + tmp = self.path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(entries, fh, indent=2) + os.replace(tmp, self.path) + + def register(self, auto_id: str, name: str, release: str = None, + shared: bool = False, purpose: str = "") -> dict: + """Record an automation (upsert by id). Shared automations store release=None.""" + entry = { + "id": auto_id, + "name": name, + "scope": "shared" if shared else "release", + "release": None if shared else release, + "purpose": purpose, + "registered_at": _now(), + } + entries = [e for e in self._load() if e.get("id") != auto_id] # upsert + entries.append(entry) + self._save(entries) + return entry + + def deregister(self, auto_id: str) -> bool: + entries = self._load() + kept = [e for e in entries if e.get("id") != auto_id] + self._save(kept) + return len(kept) != len(entries) + + def list(self, release: str = None, scope: str = None) -> list: + """List entries. `release` filters to that release's automations (scope + 'release' whose release matches). `scope` filters by 'shared'/'release'.""" + entries = self._load() + if release is not None: + entries = [e for e in entries if e.get("release") == release] + if scope is not None: + entries = [e for e in entries if e.get("scope") == scope] + return entries diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py new file mode 100644 index 00000000..a8119eda --- /dev/null +++ b/release-agent/orchestrator/render.py @@ -0,0 +1,370 @@ +"""Presentation layer — turns structured data into text/markdown. + +Kept separate from the logic (engine.py / readiness.py) so a different interface +(web UI, TUI, another frontend) can consume the same structured data and present +it its own way. These are pure functions: data in, string out. No state, no IO. +""" +from __future__ import annotations + + +# ---- readiness entry gate (the frozen table) ---- +_ICON = {"pass": "✅", "attested": "✅", "fail": "❌", "unable": "⛔", + "degraded": "⚠️", "pending": "⬜"} +_STATUS_WORD = {"pass": "PASS", "attested": "Confirmed", "fail": "FAIL", + "unable": "Unable", "degraded": "Proceeding (not silent)", + "pending": "Outstanding"} + + +def _cell(s: str) -> str: + """Single-line, pipe-safe text for a markdown table cell.""" + return (s or "").replace("\n", " ").replace("|", "\\|").strip() + + +def readiness_table(chk: dict, release_id: str) -> str: + """Canonical markdown table for the entry gate (frozen layout). + `chk` is ReadinessGate.checklist().""" + if not chk.get("items"): + return "No readiness checklist configured." + lines = [ + f"### Readiness Entry Gate — {release_id}", + "", + _cell(chk["instructions"]), + "", + "**Type legend:** `[auto]` Scout verifies · `[attest]` you confirm", + "", + "| | Type | Item | Status |", + "|---|---|---|---|", + ] + for it in chk["items"]: + box = _ICON.get(it["status"], "⬜") + typ = "`[auto]`" if it["verify"] == "auto" else "`[attest]`" + label = it.get("label") or it["id"] + if it.get("checks"): + parts = [f"[{c['name']}]({c['url']}) {'✓' if c.get('ok') else '✗'}" + if c.get("url") else f"{c['name']} {'✓' if c.get('ok') else '✗'}" + for c in it["checks"]] + detail = " · ".join(parts) + else: + detail = it.get("detail") or it.get("text") or "" + win = it.get("window") + if win and win.get("start") and win.get("end"): + detail = f"{detail} (window: {win['start']} → {win['end']})".strip() + item_cell = f"**{_cell(label)}** — {_cell(detail)}" + lines.append(f"| {box} | {typ} | {item_cell} | {_STATUS_WORD.get(it['status'], it['status'])} |") + lines.append("") + + if chk["blocked"]: + labels = [next((i["label"] for i in chk["items"] if i["id"] == b), b) + for b in chk["blocked_items"]] + lines.append("⛔ **Blocked** — cannot start: " + ", ".join(labels) + ".") + lines.append(_cell(chk["blocked_message"])) + elif chk["signed"]: + lines.append("✅ **All items satisfied — entry gate cleared.** Ready to start Phase 0.") + else: + pending = [i["label"] for i in chk["items"] if not i["satisfied"]] + lines.append("**Outstanding:** " + ", ".join(pending)) + return "\n".join(lines) + + +# ---- release status ---- +# Plain-language labels for internal engine states (never show raw state names). +_STATE_LABEL = { + "not_started": "Not started", + "running": "In progress", + "scheduled": "Scheduled — waiting for the window to open", + "awaiting_action": "Action needed from you", + "holding_gate": "Waiting for your approval", + "readiness_gate": "Entry gate — checklist pending", + "blocked": "Blocked", + "halted": "Halted", + "complete": "Complete", +} +_PHASE_ICON = {"done": "✅", "current": "⏸", "pending": "⬜", "scheduled": "🗓"} +_STEP_ICON = {"done": "✅", "gate": "⏸", "reminder": "📌", "scheduled": "🗓", + "pending": "⬜", "skipped": "⏭️"} +_STEP_STATE_WORD = {"done": "Done", "gate": "Awaiting your approval", + "reminder": "Do this — then mark done", "scheduled": "Not open yet", + "pending": "Pending", "skipped": "Skipped"} + + +def status_view(r: dict) -> str: + """Human-readable status: next-action headline → phase map → current-phase steps. + `r` is Orchestrator.status_report().""" + mode = "DRY-RUN" if r["dry_run"] else "LIVE" + bars = 20 + filled = round(bars * r["done"] / r["total"]) if r["total"] else 0 + bar = "█" * filled + "░" * (bars - filled) + label = _STATE_LABEL.get(r["status"], r["status"]) + + lines = [ + f"## Release {r['release_id']} · {mode} · {r['done']}/{r['total']} ({r['percent']}%)", + f"`{bar}`", + ] + # Code Complete Date anchor line (when known). + if r.get("ccd"): + src = {"override": "override", "manual": "confirmed", + "default": "2nd Wednesday"}.get(r.get("ccd_source"), r.get("ccd_source") or "") + srctag = f" ({src})" if src else "" + skip = " · ⚠ release SKIP set in pipeline" if r.get("skip_release") else "" + lines.append(f"**Code Complete:** {r['ccd']}{srctag} · **today:** {r.get('as_of','')}{skip}") + if r.get("ccd_conflict"): + lines.append(f"⚠ **Confirm the date** — the pipeline override is **{r['ccd_conflict']}**, " + f"which differs from the 2nd-Wednesday default (**{r['ccd']}**). " + f"Which is the real Code Complete Date — the default, or the pipeline's?") + if r.get("owner_email"): + who = (f"{r['owner_name']} " if r.get("owner_name") else "") + f"{r['owner_email']}" + lines.append(f"**Owner:** {who}") + lines.append("") + + # 1) Next-action headline — the single most important thing. + if r.get("halted"): + rsn = f" — {r['halt_reason']}" if r.get("halt_reason") else "" + lines.append(f"⛔ **HALTED**{rsn}. Nothing advances until you resume.") + elif r["blocked"]: + lines.append(f"⛔ **Blocked** — cannot start: {', '.join(r['blocked_items'])}. " + "Resolve it, or hand the release to someone who can.") + elif not r["readiness_signed"]: + lines.append("▣ **Entry gate** — the readiness checklist isn't signed yet.") + elif r.get("scheduled"): + sc = r["scheduled"] + when = _delta_phrase(sc.get("opens_in_days")) + lines.append(f"🗓 **Scheduled** — **{sc['phase_name']}** opens **{sc.get('opens','')}** " + f"({when}). Nothing to do yet.") + elif r.get("action"): + a = r["action"] + lines.append(f"📌 **Action needed** — you need to: **{a['step_name']}** " + f"(Phase {_phase_num(r, a['phase'])} · {a['phase_name']}). Mark it done when complete.") + elif r["gate"]: + g = r["gate"] + lines.append(f"⏸ **Next: your decision** — approve or deny **{g['step_name']}** " + f"(Phase {_phase_num(r, g['phase'])} · {g['phase_name']}).") + elif r["status"] == "complete": + lines.append("✔ **Release complete.** All phases done.") + elif r["status"] == "not_started": + lines.append("▶ **Not started yet** — run the next step to begin.") + else: + # in progress, between gates + nxt = _next_pending_step(r) + if nxt: + lines.append(f"▶ **In progress** — next up: **{nxt}** " + f"({r.get('current_phase_name') or ''}).") + else: + lines.append(f"▶ **{label}.**") + + # 2) Phase map (overview). + if r.get("phases"): + lines += ["", "### Phases", "| | # | Phase | Done |", "|---|---|---|---|"] + for p in r["phases"]: + icon = _PHASE_ICON.get(p["state"], "⬜") + note = "" + if p["current"]: + note = " ← you are here" + elif p["state"] == "scheduled" and p.get("opens"): + note = f" · opens {p['opens']} ({_delta_phrase(p.get('opens_in_days'))})" + lines.append(f"| {icon} | {p['num']} | {p['name']}{note} | {p['done']}/{p['total']} |") + lines.append("✅ done · ⏸ in progress · 🗓 scheduled · ⬜ not started") + + # 3) Current-phase detail (drill-down). + if r.get("current_steps"): + lines += ["", f"### ▶ Current phase — {r.get('current_phase_name','')}", + "| | Step | State |", "|---|---|---|"] + for s in r["current_steps"]: + icon = _STEP_ICON.get(s["state"], "⬜") + word = _STEP_STATE_WORD.get(s["state"], s["state"]) + tag = " 🚦" if s["gate"] else (" 📌" if s.get("reminder") else "") + lines.append(f"| {icon} | {s['name']}{tag} | {word} |") + + return "\n".join(lines) + + +def _delta_phrase(days) -> str: + if days is None: + return "" + if days == 0: + return "today" + if days == 1: + return "tomorrow" + if days == -1: + return "yesterday" + return f"in {days} days" if days > 0 else f"{-days} days ago" + + +def _phase_num(r: dict, phase_id: str): + for p in r.get("phases", []): + if p["id"] == phase_id: + return p["num"] + return "?" + + +def _next_pending_step(r: dict): + for s in r.get("current_steps", []): + if s["state"] in ("pending", "gate"): + return s["name"] + return None + + +def notification_subject(r: dict) -> str: + """Email subject for the daily phase digest (empty if nothing to send).""" + ap = r.get("active_phase") + if not ap: + return f"Release {r.get('release_id','?')} — update" + return f"Release {r.get('release_id','?')} — Phase {ap.get('num')} status" + + +def notification(r: dict) -> str: + """The DAILY PHASE DIGEST emailed to the release owner, or "" to stay silent. + `r` is Orchestrator.status_report(). The push automation sends whatever this + returns; the once-per-day cadence is enforced by the CLI (last_notified_date). + + Model (established with the user): + * Setup (readiness + CCD) is interactive in Scout — NO push. So an unsigned + release, a blocked entry gate, or a halted release stay silent here. + * The FIRST push is a phase opening (Phase 0 at CCD-7). Nothing before it + (no pre-open heads-up). + * While a phase is open with outstanding steps, report its status daily. + * Each phase gets its own digest when it opens (general pattern). + """ + rid = r.get("release_id", "?") + if (r.get("halted") or r.get("blocked") or r.get("status") == "complete" + or not r.get("readiness_signed")): + return "" # setup / paused — no push + + ap = r.get("active_phase") + if not ap or not ap.get("due"): + return "" # nothing open yet (scheduled) — no push + + head = f"Release {rid} — Phase {ap['num']}: {ap['name']}" + lines = [head] + if not ap["started"]: + opened = f" (opened {ap['opens']})" if ap.get("opens") else "" + lines.append(f"Phase {ap['num']} has opened{opened} — {ap['total']} steps to work through, none done yet.") + else: + lines.append(f"Progress: {ap['done']} of {ap['total']} steps done.") + + # What the orchestrator has already handled automatically (this phase). + completed = ap.get("completed") or [] + if completed: + lines.append(f"Completed ({len(completed)}):") + for name in completed[:8]: + lines.append(f" ✓ {name}") + + # What needs the owner right now (a live hold), then the human touchpoints ahead. + if r.get("gate"): + lines.append(f"Waiting on your decision: {r['gate']['step_name']} (approve or deny).") + elif r.get("action"): + lines.append(f"Action needed now: {r['action']['step_name']} (do it, then mark done).") + + human = [o for o in ap.get("outstanding", []) if o["gate"] or o["reminder"]] + if human: + lines.append(f"Still needs you ({len(human)}):") + for o in human[:6]: + what = "your approval" if o["gate"] else "your action" + lines.append(f" • {o['name']} — {what}") + + lines.append("Open Scout to continue the release.") + return "\n".join(lines) + + +# ---- HTML digest (nice email UX) ------------------------------------------- +# Email-safe: inline styles + table layout (Outlook-friendly), no external CSS. + +def _esc(s: str) -> str: + return (str(s or "").replace("&", "&").replace("<", "<") + .replace(">", ">").replace('"', """)) + + +# Per-status pill styling (label, text color, background). +_PILL = { + "done": ("✓ Done", "#1a7f37", "#e6f4ea"), + "now": ("⚠ Needs you now", "#b42318", "#fef3f2"), + "blocked": ("⛔ Blocked — fix & rerun", "#b42318", "#fef3f2"), + "approval": ("Your approval", "#b54708", "#fffaeb"), + "confirm": ("Your confirmation", "#b54708", "#fffaeb"), + "action": ("Your action", "#b54708", "#fffaeb"), + "auto": ("Automatic — pending", "#475467", "#f2f4f7"), +} + + +def _pill(status: str) -> str: + label, fg, bg = _PILL.get(status, _PILL["auto"]) + return (f'{label}') + + +def notification_html(r: dict) -> str: + """HTML version of the daily phase digest. Returns "" under the exact same + silence rules as notification() (reuses it as the guard). Presents EVERY step + in the active phase with a status pill, and flags what needs the owner now.""" + if not notification(r): # same silence rules / dedup gate + return "" + rid = _esc(r.get("release_id", "?")) + ap = r.get("active_phase") or {} + phase_title = _esc(f"Phase {ap.get('num')}: {ap.get('name','')}") + done, total = ap.get("done", 0), ap.get("total", 0) + pct = round(100 * done / total) if total else 0 + steps = ap.get("steps", []) + dry = r.get("dry_run") + + # The single item that needs the owner right now (the live hold), if any. + hold = r.get("gate") or r.get("action") + hold_name = _esc(hold["step_name"]) if hold else "" + hold_kind = "approve or deny" if r.get("gate") else "do it, then mark it done" + + # Rows: every step, with the active hold promoted to the "now" pill. + rows = [] + for s in steps: + st = "now" if s.get("now") else s.get("status", "auto") + name = _esc(s.get("name", "")) + star = (' ⚑' + if s.get("needs_owner") else "") + rows.append( + f'{name}{star}' + f'{_pill(st)}' + ) + rows_html = "\n".join(rows) + + attention = "" + if hold: + attention = ( + f'' + f'
' + f'⚑ Needs your attention: {hold_name} ' + f'— {hold_kind}.
' + ) + + dry_badge = ( + 'DRY-RUN' + if dry else "") + + return f"""\ +
+ + + + {attention} + + +
+
Release {rid}{dry_badge}
+
{phase_title}
+
+
Progress: {done} of {total} steps done ({pct}%)
+
+
+
+
+ + + + {rows_html} +
TaskStatus
+
+
Items marked ⚑ need you. Open Scout to continue the release.
+
+
""" + diff --git a/release-agent/orchestrator/schedule.py b/release-agent/orchestrator/schedule.py new file mode 100644 index 00000000..5c5deac6 --- /dev/null +++ b/release-agent/orchestrator/schedule.py @@ -0,0 +1,105 @@ +"""Schedule math for CCD-anchored phases — pure functions, no IO. + +The Code Complete Date (CCD) is the anchor the whole release hangs off of. +This module mirrors the logic of ADO pipeline 3038 "Code Complete Calendar +Checker" so the orchestrator resolves the *same* date the pipeline would: + + * Default : the 2nd Wednesday of the release month. + * Override: a full YYYY-MM-DD, but only if it belongs to the release month + (a stale cross-month override is ignored, exactly like the pipeline). + +Phases anchor to CCD via a spec like "CCD-7" (7 days before) or "CCD+1". +Kept IO-free so it stays deterministic and unit-testable; reading/writing the +pipeline variable lives in tools/checks.py, and CCD is stored on ReleaseState. +""" +from __future__ import annotations + +import calendar +import re +from datetime import date, timedelta +from typing import Optional + + +def parse_release_month(release_id: str) -> Tuple[int, int]: + """'2026-07' -> (2026, 7).""" + parts = release_id.split("-") + return int(parts[0]), int(parts[1]) + + +def second_wednesday(year: int, month: int) -> date: + """The 2nd Wednesday of the month (pipeline 3038's default rule).""" + weeks = calendar.monthcalendar(year, month) + wednesdays = [w[calendar.WEDNESDAY] for w in weeks if w[calendar.WEDNESDAY] != 0] + return date(year, month, wednesdays[1]) + + +def parse_date(s: Optional[str]) -> Optional[date]: + """Parse 'YYYY-MM-DD' -> date, or None if empty/invalid.""" + if not s or not str(s).strip(): + return None + try: + y, m, d = (int(x) for x in str(s).strip().split("-")) + return date(y, m, d) + except (ValueError, TypeError): + return None + + +def default_ccd(release_id: str) -> date: + """The canonical CCD for a release month: the 2nd Wednesday. This is the + source of truth — a differing pipeline override is treated as a *question* + to confirm, not a value to adopt silently.""" + year, month = parse_release_month(release_id) + return second_wednesday(year, month) + + +def pipeline_conflict(release_id: str, override: Optional[str], stored_ccd: Optional[str] = None): + """Return the pipeline override date IF it is a valid in-month date that + DIFFERS from our reference CCD — i.e. a divergence the user must resolve. + Otherwise None (override empty, cross-month, or already in agreement). + + The reference is `stored_ccd` when provided (what this release is anchored + to), else the 2nd-Wednesday default (used at init before anything is stored). + """ + od = parse_date(override) + if not od: + return None + year, month = parse_release_month(release_id) + if (od.year, od.month) != (year, month): + return None # month-scoped, like the pipeline + reference = parse_date(stored_ccd) or default_ccd(release_id) + return od if od != reference else None + + +_ANCHOR_RE = re.compile(r"^CCD\s*([+-]\s*\d+)?$", re.IGNORECASE) + + +def anchor_offset(spec: str) -> int: + """'CCD-7' -> -7, 'CCD+1' -> 1, 'CCD' -> 0.""" + m = _ANCHOR_RE.match((spec or "").strip()) + if not m: + raise ValueError(f"bad anchor spec: {spec!r} (expected e.g. 'CCD-7')") + grp = m.group(1) + return int(grp.replace(" ", "")) if grp else 0 + + +def anchor_date(ccd: date, spec: str) -> date: + """The calendar date a phase with `spec` opens, given CCD.""" + return ccd + timedelta(days=anchor_offset(spec)) + + +def today() -> date: + """'Now' at date granularity. `--as-of` overrides this for testable dry-runs.""" + return date.today() + + +def humanize_delta(days: int) -> str: + """'in 3 days' / 'today' / '2 days ago' — for countdowns.""" + if days == 0: + return "today" + if days == 1: + return "tomorrow" + if days == -1: + return "yesterday" + if days > 0: + return f"in {days} days" + return f"{-days} days ago" diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py new file mode 100644 index 00000000..4d08006a --- /dev/null +++ b/release-agent/orchestrator/state.py @@ -0,0 +1,112 @@ +"""Release Orchestrator — run-state model (X5). + +Two kinds of state, per the architecture: + * DERIVED : recomputed from systems of record (ADO/Git/Play Console/ADX). Never stored here. + * PERSISTED: decisions/intent, step completion, pending human actions, notes. + Stored in release-state.json (the Release State Record). + +This module owns ONLY the persisted state. Reconcile-on-resume (deriving live +state) is a separate concern handled by tools/reconcile.py (stubbed for now). +""" +from __future__ import annotations +import json +import os +from dataclasses import dataclass, field, asdict +from datetime import datetime, timezone +from typing import Optional + + +SCHEMA_VERSION = 1 + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +@dataclass +class StepState: + """Persisted state for a single step.""" + status: str = "pending" # pending | done | skipped | blocked + completed_at: Optional[str] = None + note: Optional[str] = None + by: Optional[str] = None # 'agent' (stub) or 'human' + + +@dataclass +class GateDecision: + """A recorded human decision at a gate (audit trail).""" + step: str + decision: str # approved | denied | held + at: str + by: str = "human" + comment: Optional[str] = None + + +@dataclass +class ReleaseState: + """The Release State Record — one per monthly release.""" + schema_version: int = SCHEMA_VERSION + release_id: str = "" # e.g. 2026-07 + created_at: str = field(default_factory=_now) + updated_at: str = field(default_factory=_now) + dry_run: bool = True + # Release owner — the engineer running this release (release metadata). The + # push reminders email this address; resolved from the signed-in user at init. + owner_email: Optional[str] = None + owner_name: Optional[str] = None + # Code Complete Date — the anchor the phases hang off of (orchestrator's truth, + # seeded from / written back to pipeline 3038). ccd is 'YYYY-MM-DD'. + ccd: Optional[str] = None + ccd_source: Optional[str] = None # 'default' (2nd Wed) | 'override' | 'manual' + ccd_conflict: Optional[str] = None # a pipeline override date that DIFFERS from ccd (unresolved) + skip_release: bool = False # mirrors the pipeline 'skipRelease' switch (display) + # readiness entry gate (must be signed before Phase 0) + readiness_signed: bool = False + readiness_signed_at: Optional[str] = None + readiness_items: dict = field(default_factory=dict) # item_id -> {status,...} + blocked: bool = False # an item was declared unsatisfiable + blocked_items: list = field(default_factory=list) + # manual overrides (human-driven transitions, §7.1) + halted: bool = False # emergency hold + halt_reason: Optional[str] = None + # cursor + current_phase: Optional[str] = None + current_step: Optional[str] = None + status: str = "not_started" # not_started | running | scheduled | awaiting_action | holding_gate | halted | blocked | complete + # persisted detail + steps: dict = field(default_factory=dict) # "phase.step" -> StepState (as dict) + gate_decisions: list = field(default_factory=list) + pending_human: list = field(default_factory=list) # outstanding human actions + last_notified: Optional[str] = None # last push message (legacy; kept for load compat) + last_notified_date: Optional[str] = None # YYYY-MM-DD of the last daily digest sent + notes: list = field(default_factory=list) + + # ---- persistence ---- + @classmethod + def load(cls, path: str) -> "ReleaseState": + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + return cls(**data) + + def save(self, path: str) -> None: + self.updated_at = _now() + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(asdict(self), fh, indent=2) + os.replace(tmp, path) + + # ---- step helpers ---- + @staticmethod + def key(phase: str, step: str) -> str: + return f"{phase}.{step}" + + def get_step(self, phase: str, step: str) -> StepState: + raw = self.steps.get(self.key(phase, step)) + return StepState(**raw) if raw else StepState() + + def set_step(self, phase: str, step: str, state: StepState) -> None: + self.steps[self.key(phase, step)] = asdict(state) + + def is_done(self, phase: str, step: str) -> bool: + return self.get_step(phase, step).status in ("done", "skipped") diff --git a/release-agent/phases/__init__.py b/release-agent/phases/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release-agent/phases/agents/__init__.py b/release-agent/phases/agents/__init__.py new file mode 100644 index 00000000..ce2f9be8 --- /dev/null +++ b/release-agent/phases/agents/__init__.py @@ -0,0 +1,38 @@ +"""Phase-agent registry — aggregates every phase's agents into one lookup. + +Each phase's real agents live in `phases/agents/.py`, which exposes a +module-level `REGISTRY = {agent_id: run(phase_id, step, dry_run, state) -> StepResult}`. +This package merges them all into a single `REGISTRY` so the engine does ONE +lookup, and guards against two phases claiming the same agent id. + +Adding a phase's agents is a one-line change: create `phases/agents/.py` +with a `REGISTRY`, then add its name to `_PHASE_MODULES` below. The engine never +changes — it already looks up the merged registry. +""" +from __future__ import annotations + +from importlib import import_module + +from phases.stub_runner import get_runner as _stub_get_runner + +# Phase agent modules, in phase order. Add a new phase's module name here. +_PHASE_MODULES = [ + "preflight", +] + +REGISTRY: dict = {} +for _name in _PHASE_MODULES: + _mod = import_module(f"{__name__}.{_name}") + for _agent_id, _runner in getattr(_mod, "REGISTRY", {}).items(): + if _agent_id in REGISTRY: + raise RuntimeError( + f"duplicate phase-agent id '{_agent_id}': phases.agents.{_name} " + f"collides with an agent already registered by an earlier phase") + REGISTRY[_agent_id] = _runner + + +def get_runner(agent_id: str): + """The ONE dispatch seam the engine uses: return the real phase agent for + `agent_id` if one is registered, otherwise the stub runner (which handles + unbuilt steps). Always returns a callable with the run(...) contract.""" + return REGISTRY.get(agent_id) or _stub_get_runner(agent_id) diff --git a/release-agent/phases/agents/preflight.py b/release-agent/phases/agents/preflight.py new file mode 100644 index 00000000..d7d41d55 --- /dev/null +++ b/release-agent/phases/agents/preflight.py @@ -0,0 +1,311 @@ +"""Real Phase-0 pre-flight agents (deterministic; az CLI / HTTP). + +These replace the `stub` for specific Phase-0 steps with genuine actions: + + * breaking_detect (step `breaking`, S2) — reads the common-for-android + changelog, finds breaking ([MAJOR]) changes in the unreleased section, + and drafts the OneAuth comms. Read-only. + * wiki_payload (step `wiki`, S0) — creates the per-release payload + wiki subpage under the standing history page. A real ADO write. + +Contract (shared with the stub): run(phase_id, step, dry_run, state) -> StepResult + +Dry-run safety: in a dry-run release every agent SIMULATES — no network, no +writes — so tests and dry-run rehearsals never touch production. Only a real +(dry_run=False) release performs the action. +""" +from __future__ import annotations +from urllib import request as _request + +from phases.stub_runner import StepResult + + +# ---- config ---------------------------------------------------------------- +def _load_cfg() -> dict: + """Phase-0 config (config/preflight.yaml), via the shared per-phase loader.""" + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight") + + +def _fetch_text(url: str, timeout: int = 20) -> str: + req = _request.Request(url, headers={"User-Agent": "release-agent-preflight/1.0"}) + with _request.urlopen(req, timeout=timeout) as resp: + return resp.read().decode("utf-8", "replace") + + +# ---- breaking-change detection (pure, unit-tested) ------------------------- +def parse_breaking(changelog_text: str, section: str = "vNext", + tag: str = "[MAJOR]") -> list: + """Return the list of `tag` (breaking) entry lines inside `section`. + + The changelog is a flat text file: a section header line (e.g. "vNext"), + an underline, then `- [SEVERITY] ... (#PR)` bullets, until the next + "Version X.Y.Z" header. We scan only the requested section. + """ + entries, in_section = [], False + for raw in changelog_text.splitlines(): + s = raw.strip() + if not in_section: + if s == section: + in_section = True + continue + if s.startswith("Version "): + break + if tag in raw: + entries.append(s) + return entries + + +def _draft_breaking_comms(entries: list, state=None) -> str: + release = getattr(state, "release_id", None) or "this release" + bullets = "\n".join(f"- {e}" for e in entries) + return ( + f"Subject: [Action] Breaking OneAuth changes in {release}\n\n" + f"Hi OneAuth team,\n\n" + f"The upcoming Android common release ({release}) contains the following " + f"breaking change(s). Please review for downstream impact before code " + f"complete:\n\n{bullets}\n\n" + f"Thanks,\nRelease Orchestrator" + ) + + +def run_breaking(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + cfg = _load_cfg().get("breaking", {}) + url = cfg.get("changelog_url") + section = cfg.get("section", "vNext") + tag = cfg.get("breaking_tag", "[MAJOR]") + if dry_run: + return StepResult( + True, + f"[dry-run] Would scan the '{section}' changelog section for {tag} " + f"(breaking) entries and draft OneAuth comms.", + "agent", + ) + if not url: + return StepResult(False, "breaking: no changelog_url configured", "agent") + try: + text = _fetch_text(url) + except Exception as e: # noqa: BLE001 - network/parse errors -> hold for human + return StepResult(False, f"breaking: could not fetch changelog ({e})", "agent") + entries = parse_breaking(text, section, tag) + if not entries: + return StepResult( + True, f"No breaking ({tag}) changes in '{section}' — no OneAuth comms needed.", + "agent", + ) + listing = "\n".join(f" - {e}" for e in entries) + draft = _draft_breaking_comms(entries, state) + return StepResult( + True, + f"Detected {len(entries)} breaking ({tag}) change(s) in '{section}':\n{listing}\n\n" + f"--- DRAFT COMMS (send to OneAuth) ---\n{draft}", + "agent", + ) + + +# ---- payload wiki subpage -------------------------------------------------- +def _payload_template(state=None) -> str: + release = getattr(state, "release_id", None) or "unknown" + ccd = getattr(state, "ccd", None) or "TBD" + owner = getattr(state, "owner_name", None) or getattr(state, "owner_email", None) or "TBD" + return ( + f"# {release} — Release Payload\n\n" + f"| Field | Value |\n|---|---|\n" + f"| Release | {release} |\n| Code Complete Date | {ccd} |\n| Release owner | {owner} |\n\n" + f"## Built versions\n\n" + f"_Filled during Build & Lib Verification (Phase 2)._\n\n" + f"| Artifact | Version |\n|---|---|\n| | |\n" + ) + + +def _page_name(state, n: int = 1) -> str: + """Payload page name: ' Release', e.g. 'August 2026 Release'. + A numbered variant ('August 2026 2 Release') is used when a page already + exists for the month (n >= 2).""" + import calendar + release = getattr(state, "release_id", None) or "unknown" + try: + year, month = release.split("-")[:2] + base = f"{calendar.month_name[int(month)]} {int(year)}" + except Exception: # noqa: BLE001 - fall back to the raw id + base = release + return f"{base} {n} Release" if n and n >= 2 else f"{base} Release" + + +def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + cfg = _load_cfg().get("wiki", {}) + org = cfg.get("org") + project = cfg.get("project") + wiki = cfg.get("wiki") + parent = (cfg.get("parent_path") or "").rstrip("/") + base_name = _page_name(state) + base_path = f"{parent}/{base_name}" + if dry_run: + return StepResult( + True, + f"[dry-run] Would create payload wiki subpage '{base_name}' under " + f"'{parent}' (duplicate-safe: a second numbered page if it already exists).", + "agent", + ) + if not (org and project and wiki and parent): + return StepResult(False, "wiki: incomplete configuration", "agent") + from tools.checks import create_wiki_page, wiki_page_exists + + # Duplicate handling: if the month's page already exists, DON'T overwrite — + # notify and create the next free " N Release" page instead. + if wiki_page_exists(org, project, wiki, base_path): + n = 2 + while n <= 50: + cand_name = _page_name(state, n) + cand_path = f"{parent}/{cand_name}" + if not wiki_page_exists(org, project, wiki, cand_path): + res = create_wiki_page(org, project, wiki, cand_path, _payload_template(state)) + if not res.ok: + return StepResult(False, f"wiki: could not create '{cand_path}' — {res.detail}", "agent") + return StepResult( + True, + f"⚠ A payload page already exists for this month ('{base_name}'). " + f"Left it untouched and created a SECOND page: '{cand_name}'. ({res.detail})", + "agent", + ) + n += 1 + return StepResult(False, f"wiki: too many existing pages for '{base_name}'", "agent") + + res = create_wiki_page(org, project, wiki, base_path, _payload_template(state)) + if not res.ok: + return StepResult(False, f"wiki: could not create '{base_path}' — {res.detail}", "agent") + return StepResult(True, f"Payload wiki subpage ready: '{base_name}' ({res.detail})", "agent") + + +# ---- Component Governance alerts (report-only) ----------------------------- +def _cg_summary(alerts: list, high_sev: list): + """Return (active, high) lists from raw alerts. `high` = active alerts whose + severity is in high_sev (critical/high).""" + active = [a for a in alerts if (a.get("alertState") or "").lower() == "active"] + high = [a for a in active if (a.get("severity") or "").lower() in + [s.lower() for s in high_sev]] + return active, high + + +def _cg_report(active: list, high: list) -> str: + from collections import Counter + by_sev = Counter((a.get("severity") or "unknown").lower() for a in active) + counts = ", ".join(f"{n} {s}" for s, n in sorted(by_sev.items(), + key=lambda kv: (-kv[1], kv[0]))) or "none" + lines = [f"Component Governance: {len(active)} active alert(s) — {counts}."] + if high: + lines.append(f"High/Critical ({len(high)}) — review before release:") + for a in high[:15]: + comp = (a.get("component") or {}).get("displayName") or "" + ver = (a.get("component") or {}).get("displayVersion") or "" + rec = (a.get("actionItems") or "").strip().split("\n")[0] + title = a.get("title") or a.get("summary") or "?" + sev = (a.get("severity") or "").capitalize() + comp_str = f" — {comp} {ver}".rstrip() if comp else "" + lines.append(f" • [{sev}] {title}{comp_str}" + (f" — {rec}" if rec else "")) + else: + lines.append("No High/Critical active alerts.") + return "\n".join(lines) + + +def run_cg_alerts(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + """Report active Component Governance alerts. Passes when there are no active + High/Critical alerts; BLOCKS (holds) when there are — the owner must fix the + issues and RERUN this step (re-checks), or override by skipping it.""" + cfg = _load_cfg().get("cg", {}) + if dry_run: + return StepResult( + True, + "[dry-run] Would query Component Governance for active alerts and " + "report counts + High/Critical items (blocking on High/Critical).", + "agent", + ) + required = ("resource", "governance_host", "project_id", "governed_repo_id", "branch") + if not all(cfg.get(k) for k in required): + return StepResult(False, "cg: incomplete configuration", "agent") + from tools.checks import fetch_cg_alerts + ok, alerts, detail = fetch_cg_alerts( + cfg["resource"], cfg["governance_host"], cfg["project_id"], + cfg["governed_repo_id"], cfg["branch"]) + if not ok: + return StepResult(False, f"cg: could not read alerts — {detail}", "agent") + active, high = _cg_summary(alerts, cfg.get("high_severities", ["critical", "high"])) + report = _cg_report(active, high) + if high: + # Block: the report is shown, the step holds. The owner fixes the alerts + # and reruns this step (re-checks), or skips to override. + return StepResult( + False, + report + "\n→ Fix the High/Critical alerts (or wait for remediation), then " + "RERUN this step to re-check — or skip to override with a reason.", + "agent", + ) + return StepResult(True, report, "agent") + + +# ---- Calendar Checker schedule verification -------------------------------- +def _iso_age_days(iso: str): + """Whole days between an ISO-8601 timestamp and now (UTC), or None if unparseable.""" + from datetime import datetime, timezone + if not iso: + return None + try: + s = iso.replace("Z", "+00:00") + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return (datetime.now(timezone.utc) - dt).days + except ValueError: + return None + + +def run_cron_check(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + """Verify the Calendar Checker pipeline is scheduled AND firing, by confirming a + recent `schedule`-reason run. Passes if a scheduled run is within the staleness + window; BLOCKS (fix + rerun, or skip) if there's none or it's stale.""" + cfg = _load_cfg().get("cron", {}) + name = cfg.get("name", "Calendar Checker") + if dry_run: + return StepResult( + True, + f"[dry-run] Would verify '{name}' (pipeline {cfg.get('pipeline_id')}) has a " + f"recent scheduled run.", + "agent", + ) + if not all(cfg.get(k) for k in ("pipeline_id", "org", "project")): + return StepResult(False, "cron: incomplete configuration", "agent") + from tools.checks import latest_scheduled_build + ok, run, detail = latest_scheduled_build(cfg["org"], cfg["project"], cfg["pipeline_id"]) + if not ok: + return StepResult(False, f"cron: could not read build history — {detail}", "agent") + if not run: + return StepResult( + False, + f"{name}: no scheduled run found in recent history — the cron may be " + f"disabled. Investigate, then rerun this step (or skip to override).", + "agent", + ) + age = _iso_age_days(run.get("queueTime")) + max_stale = cfg.get("max_staleness_days", 2) + when = (run.get("queueTime") or "")[:16] + if age is not None and age > max_stale: + return StepResult( + False, + f"{name}: last scheduled run was {when} ({age}d ago) — stale (> {max_stale}d). " + f"The schedule may be broken. Fix + rerun this step, or skip to override.", + "agent", + ) + return StepResult( + True, + f"{name} is scheduled and firing — last scheduled run {when} ({run.get('result')}).", + "agent", + ) + + +# ---- registry -------------------------------------------------------------- +REGISTRY = { + "breaking_detect": run_breaking, + "wiki_payload": run_wiki, + "cg_alerts": run_cg_alerts, + "cron_check": run_cron_check, +} diff --git a/release-agent/phases/readiness_verifiers.py b/release-agent/phases/readiness_verifiers.py new file mode 100644 index 00000000..d5d7a0eb --- /dev/null +++ b/release-agent/phases/readiness_verifiers.py @@ -0,0 +1,87 @@ +"""Readiness AUTO verifiers. + +An auto verifier must FULLY verify its item — it returns pass or fail, never a +half-measure. If something cannot be fully proven programmatically, it must NOT +be an auto item (make it an attest item in readiness.yaml instead). + +Contract: verify(item, dry_run) -> VerifyResult(status, message) + status: "pass" | "fail" +""" +from __future__ import annotations +from dataclasses import dataclass +import os + + +@dataclass +class VerifyResult: + status: str # "pass" | "fail" + message: str + details: list = None # optional per-check breakdown: [{name, url, ok, detail}] + + @property + def ok(self) -> bool: + return self.status == "pass" + + +def verify_build_defs(item: dict, dry_run: bool) -> VerifyResult: + """Confirm the engineer can access every configured ADO build definition, + using `az pipelines build definition show`. Fully verified access — pass/fail. + Returns per-check details (name, url, ok) so the display can link each one.""" + from tools.checks import check_ado_build_def + + checks = [c for c in item.get("checks", []) if c.get("type") == "ado_build_def"] + if not checks: + return VerifyResult("fail", "no build definitions configured to check", []) + details, any_fail = [], False + for c in checks: + r = check_ado_build_def(c["org"], c["project"], c["id"]) + details.append({"name": c.get("name", str(c["id"])), "url": c.get("url"), + "ok": r.ok, "detail": r.detail}) + if not r.ok: + any_fail = True + msg = "; ".join(f"{'OK' if d['ok'] else 'FAIL'} {d['name']}" for d in details) + return VerifyResult("fail" if any_fail else "pass", msg, details) + + +def verify_mcp_servers(item: dict, dry_run: bool) -> VerifyResult: + """Confirm every MCP server the skill needs (ICM, Kusto/ADX) is registered in + Scout's config. Reuses the infra preflight in READ-ONLY mode (register=False) + against config/requirements.yaml — the single source of truth for MCP deps. + Fully verified: pass only if all are `present`, else fail listing the missing + ones (fix = run bootstrap / `infra --register`, then RESTART Scout).""" + from orchestrator import infra + + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + req_path = os.path.join(root, "config", "requirements.yaml") + try: + req = infra.load_requirements(req_path) + except OSError as e: + return VerifyResult("fail", f"cannot read requirements.yaml: {e}", []) + results = infra.ensure_mcp_servers(req, register=False) + if not results: + return VerifyResult("fail", "no MCP servers configured to check", []) + details, missing = [], [] + for r in results: + ok = r.get("status") == "present" + details.append({"name": r.get("name", r.get("scout_key")), "url": None, + "ok": ok, "detail": r.get("detail", "")}) + if not ok: + missing.append(r.get("scout_key") or r.get("id")) + if missing: + return VerifyResult( + "fail", + "not registered: " + ", ".join(missing) + + " — run bootstrap (or `python -m orchestrator.cli infra`), then RESTART Scout", + details) + return VerifyResult( + "pass", "registered: " + ", ".join(r.get("scout_key", "?") for r in results), details) + + +REGISTRY = { + "build_defs": verify_build_defs, + "mcp_servers": verify_mcp_servers, +} + + +def get_verifier(verifier_id: str): + return REGISTRY.get(verifier_id) diff --git a/release-agent/phases/stub_runner.py b/release-agent/phases/stub_runner.py new file mode 100644 index 00000000..cb61c9f3 --- /dev/null +++ b/release-agent/phases/stub_runner.py @@ -0,0 +1,47 @@ +"""Stub phase runner — the fallback for steps that don't yet have a real agent. + +Phase 0 has real agents (phases/agents/preflight.py); every other step still +maps to `agent: stub`. The stub does NOT perform the real action — it returns a +mock result telling the conductor what a human would do, so the end-to-end flow +can be driven and tested before each real agent exists. + +When a real phase agent is built, it implements the same contract: + run(phase_id, step, dry_run, state) -> StepResult +and replaces the stub for that step's `agent` id (registered in phases/agents/). +""" +from __future__ import annotations +from dataclasses import dataclass + + +@dataclass +class StepResult: + ok: bool + action: str # human-readable description of what happened / should happen + by: str # 'agent' (stub did it) or 'human' (needs a person) + + +def run_stub(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + """Mock action for a step. Agent-owned steps are 'auto-completed' (mock); + human-owned steps return a reminder that a person must act.""" + owner = step.get("owner", "agent") + name = step.get("name", step["id"]) + if owner == "human": + return StepResult( + ok=True, + action=f"[STUB] Reminder — a human must: {name}", + by="human", + ) + prefix = "[STUB/dry-run]" if dry_run else "[STUB]" + return StepResult( + ok=True, + action=f"{prefix} Would run agent for: {name} (mock success)", + by="agent", + ) + + +# Registry: maps an agent id -> runner. For now everything is the stub. +REGISTRY = {"stub": run_stub} + + +def get_runner(agent_id: str): + return REGISTRY.get(agent_id, run_stub) diff --git a/release-agent/setup/bootstrap.ps1 b/release-agent/setup/bootstrap.ps1 new file mode 100644 index 00000000..44a53189 --- /dev/null +++ b/release-agent/setup/bootstrap.ps1 @@ -0,0 +1,86 @@ +<# +.SYNOPSIS + One-time setup for the Release Orchestrator (/release-agent) on this machine. + Small by design: it only prepares Scout + the environment so the real work + can run inside Scout. It does NOT run a release. + +.DESCRIPTION + Steps: + 1. Infrastructure preflight — check CLIs/host deps AND register + verify the + MCP servers the skill needs inside Scout (from config/requirements.yaml). + 2. Install the /release-agent skill into the Scout skills folder. + 3. Print next steps. + +.EXAMPLE + pwsh ./setup/bootstrap.ps1 +#> +[CmdletBinding()] +param( + [string]$ScoutSkillsDir = "$env:USERPROFILE\.scout\m-skills", + [switch]$SkipSkillInstall +) + +$ErrorActionPreference = "Stop" +$AgentRoot = Split-Path -Parent $PSScriptRoot # release-agent/ +$RepoRoot = Split-Path -Parent $AgentRoot # android-complete/ +$ReqFile = Join-Path $AgentRoot "config\requirements.yaml" + +Write-Host "Release Orchestrator bootstrap`n" -ForegroundColor Cyan + +# ---- 1. Infrastructure preflight (CLIs + MCP servers), data-driven ---- +# Delegates to the engine (python -m orchestrator.cli infra), which reads +# config/requirements.yaml, checks every CLI/host dependency, and REGISTERS any +# missing MCP servers into Scout's config (backing it up first). One home for the +# logic; bootstrap just needs python+pyyaml to call it. +Write-Host "1. Infrastructure preflight (from config/requirements.yaml)" +if (-not (Test-Path $ReqFile)) { Write-Host " requirements.yaml not found at $ReqFile" -ForegroundColor Red; exit 1 } + +$haveInfra = $false +try { python -c "import yaml" 2>$null; if ($LASTEXITCODE -eq 0) { $haveInfra = $true } } catch { $haveInfra = $false } + +$ok = $true +$restartNeeded = $false +if (-not $haveInfra) { + Write-Host " [ ] Python + PyYAML ... MISSING (needed to run the preflight)" -ForegroundColor Yellow + Write-Host " install: Python 3.9+ then python -m pip install pyyaml" -ForegroundColor DarkYellow + $ok = $false +} else { + Push-Location $AgentRoot + try { + $out = python -m orchestrator.cli infra 2>&1 + $out | ForEach-Object { Write-Host " $_" } + if ($LASTEXITCODE -ne 0) { $ok = $false } + if ($out -match "RESTART Scout") { $restartNeeded = $true } + } finally { Pop-Location } + + # engine config presence (cheap local sanity check) + Write-Host -NoNewline " [ ] engine config (phases.yaml) ... " + if (Test-Path (Join-Path $AgentRoot 'config\phases.yaml')) { Write-Host "OK" -ForegroundColor Green } + else { Write-Host "MISSING" -ForegroundColor Yellow; $ok = $false } +} + +if (-not $ok) { + Write-Host "`nSome infrastructure is missing — resolve the items above, then re-run." -ForegroundColor Yellow +} +if ($restartNeeded) { + Write-Host "`n>>> RESTART Scout now so the newly-registered MCP server(s) load. <<<" -ForegroundColor Cyan +} + +Write-Host "`n2. Skill install" +if ($SkipSkillInstall) { + Write-Host " Skipped (--SkipSkillInstall)." +} else { + $src = Join-Path $AgentRoot "skill\SKILL.md" + $destDir = Join-Path $ScoutSkillsDir "release-agent" + New-Item -ItemType Directory -Force -Path $destDir | Out-Null + Copy-Item $src (Join-Path $destDir "SKILL.md") -Force + Write-Host " Installed /release-agent skill -> $destDir" -ForegroundColor Green +} + +Write-Host "`n3. Next steps" -ForegroundColor Cyan +Write-Host " * Open Scout and run: /release-agent" +Write-Host " * Or drive the engine directly from $AgentRoot :" +Write-Host " python -m orchestrator.cli init --release 2026-07" +Write-Host " python -m orchestrator.cli next --release 2026-07" +Write-Host " python -m orchestrator.cli status --release 2026-07" +Write-Host "`n Everything runs in DRY-RUN by default. Nothing touches production.`n" diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md new file mode 100644 index 00000000..39506dfa --- /dev/null +++ b/release-agent/skill/SKILL.md @@ -0,0 +1,282 @@ +--- +name: release-agent +description: Drive an Android release end-to-end using the Release Orchestrator backbone. Use when the user invokes /release-agent, says "start a release", "continue the release", "advance the release", "approve the gate", "release status", or asks about release run-state. The engine is deterministic and does the real work; this skill is the conversation layer that discovers releases, presents gate briefs and status, and relays the human decision. +--- + +# /release-agent — Release Orchestrator conductor + +You are the conversation layer over the **Release Orchestrator engine** (deterministic Python). +The engine decides what happens next; you discover releases, present status/gates nicely, and relay decisions. +**Never decide the release flow yourself, and never invent a release** — always call the engine. + +## Where things live +- Engine + config: `C:\repos\android-complete\release-agent\` (run commands from here). +- Run-state: `C:\repos\android-complete\.release-runs\\release-state.json` (gitignored; one per month, e.g. `2026-07`). +- The `setup/bootstrap.ps1` script ONLY prepares the machine. Its first step is an **infrastructure preflight** (`python -m orchestrator.cli infra`): it checks the CLIs/host deps in `config/requirements.yaml` AND registers the **MCP servers** the skill needs into Scout's config — the **ICM** server (on-call lookups) and the **Kusto/ADX** server (telemetry queries), both provided by the Agency CLI — backing the config up and telling the engineer to restart Scout. It also checks **Scout itself is installed** (`~/.scout`); if not, it stops and says to install Scout first. It does **not** start a release — that happens here, in Scout. +- **Kusto is multi-cluster:** clusters live as data under `kusto_clusters` in `config/requirements.yaml`; infra wires them all into the one Kusto MCP via `--known-services`. To make a new cluster queryable, add an entry there and re-run `python -m orchestrator.cli infra` (then restart Scout). +- If an infra check ever fails (a needed MCP server isn't registered, or Scout wasn't restarted after registering), run `python -m orchestrator.cli infra` and tell the user to restart Scout; the manifest is `config/requirements.yaml`. + +## ALWAYS discover first (the none / one / many rule) +On ANY request about a release (status, continue, approve, advance), **do not assume a release id**. +First run discovery and branch on the result: + +``` +python -m orchestrator.cli list --json +``` + +The JSON has `resolution`: +- **`none`** → there is NO active release on this machine. Tell the user briefly, then **use the `m_ask_user` prompt tool** (not a free-text question) to offer starting one — see "Starting a release" below. Only run `init` after they choose. +- **`one`** → use that release (`release.release_id`). Proceed. +- **`ambiguous`** (several exist) → present the list from `all`, then **use `m_ask_user`** to let the user pick which release to act on (one option per release id, most-recent first). Do not act until they choose. +- **`explicit`** (you passed `--release` and it matched) → use it. + +Never run `status`/`next`/`approve` against a release id you haven't confirmed exists via `list`. + +## Starting a release (don't make the user type a date format) + +The release id is just `YYYY-MM`. **You compute it — never ask the user to type the format.** +Work out the current month from today's date (e.g. today 2026-07 → `2026-07`). + +When no release is active (or the user says "start a release"), call the **`m_ask_user`** prompt tool with clickable options, e.g.: +- **"Current month (``)"** ← recommended +- **"A different month"** + +If they pick the current month, run `init --release ` immediately. +If they pick "a different month", then (and only then) ask which month in a follow-up `m_ask_user` free-text prompt (hint: "e.g. next month, or 2026-08") and convert whatever they say into `YYYY-MM` yourself. Accept natural answers ("this month", "next month", "August") — do the date math for them; don't demand a rigid format. +Default to **dry-run**; only pass `--live` if the user explicitly asks for a live run. + +`init` records the **release owner** (the engineer running it) in the release metadata — resolved from the signed-in `az` user, and reminders are emailed to that address. You can pass a richer profile with `--owner-email`/`--owner-name` (e.g. from `workiq_get_my_profile`), or change it later with `set-owner`. Never hardcode a recipient. + +### Ensure push reminders exist (per release — provisioned at start, torn down at close) + +Right after `init`, make sure the **push-reminder automation** exists for THIS release so reminders reach the user even with Scout closed. It is a **per-release** automation: created when the release starts and removed when it closes (see teardown below). +1. List automations (`m_list_automations`). If one named **"Release push reminders"** already exists AND the registry has it scoped to the current release (`automation list --release --json`), **leave it** — don't duplicate. +2. If it's missing, create it with `m_create_automation`: + - **name:** `Release push reminders` + - **schedule:** `every hour` + - **teamsNotify:** `never` + - **prompt:** from `C:\repos\android-complete\release-agent` run `python -m orchestrator.cli tick --json` (this ADVANCES the active release — running the agent steps that can run, holding at gates/actions — then returns `{message, html, subject, owner_email, owner_name, release}`); if `message` is non-empty and `owner_email` is set, email it via `workiq_send_email` (`to: [owner_email]`, `subject:` the `subject` value, `body:` the `html` value with `isHtml: true` — fall back to the plain `message` with `isHtml: false` only if `html` is empty); if `message` is empty, do nothing. (Recipient comes from `owner_email` — never hardcode. Do **not** use `m_send_teams_message` (bot relay 404s) or the Teams self-chat (delivers silently).) +3. **Register it to this release** so it's tracked and torn down at close: + `python -m orchestrator.cli automation register --id --name "Release push reminders" --release --purpose "hourly advance + phase digest email to owner"` + +Do it silently as part of the start flow (the user already opted into push); don't re-ask each release. (The automation runs `tick` in discovery mode, so it targets the active release automatically.) **Why hourly, not once at 9am:** `tick` is idempotent (advancing is a no-op once holding at a gate, and the digest de-dupes to one email per calendar day), so running it every hour means a run missed while the machine was off — e.g. the 9am tick — is simply picked up by the next tick after the machine is on. A single daily trigger would be skipped for that day. + +### Any automation you provision MUST be registered (for teardown) + +Whenever you create a Scout automation for the orchestrator, immediately record it with `automation register` so nothing gets orphaned: +- **Per-release** (the normal case, e.g. push reminders, a phase watcher) → `--release `. **Removed when that release closes.** +- **Shared / persistent** (rare — only something genuinely meant to outlive every release) → `--shared` (no `--release`). Not torn down at close. Default to per-release unless there's a clear reason. + +At **release close** (status complete, the Release Close phase, or the user asks to "clean up automations"), tear down that release's automations: +1. `python -m orchestrator.cli automation list --release --json` — the automations provisioned for this release. +2. For each entry, delete the real Scout automation with `m_delete_automation` (id from the entry), then `python -m orchestrator.cli automation deregister --id `. +3. Shared automations (if any) are **not** in the release-scoped list, so they survive — leave them. +Confirm with the user before deleting, and report what was removed. + +## Code Complete Date (CCD) & phase scheduling + +Phases are **anchored to the Code Complete Date**, not started on demand. **The CCD is the 2nd Wednesday of the release month — that's the canonical default.** `init` computes it and prints when Phase 0 opens. + +`init` also *reads* the pipeline (ADO 3038 `overrideCodeCompleteDate`) but **does not silently adopt it.** If the pipeline holds a **different in-month date**, that's a **conflict to resolve, not an answer**: the status view shows a *"⚠ Confirm the date"* line and `status --json` sets `ccd_conflict`. When you see a conflict, **ask the user which is the real CCD** via `m_ask_user`, e.g.: +- **"Use the 2nd-Wednesday default (``)"** — then offer to sync the pipeline: run `set-ccd --release --default --reason ""` (preview) → show it → `--confirm` to clear the pipeline override so they match. +- **"Use the pipeline date (``)"** — run `set-ccd --release --date --reason "confirmed CCD is " --confirm` (stores it locally; the pipeline already has it). + +Either resolution clears the conflict. Never pick for the user. + +- **Phase 0 (Pre-flight) opens at CCD‑7** (7 days before CCD). You can `init` any time, but until CCD‑7 the release sits in **`scheduled`** — the engine runs nothing. The status view says *"🗓 Scheduled — Pre‑flight opens `` (in N days). Nothing to do yet."* Relay that plainly; don't try to force it forward. +- When the clock reaches CCD‑7, `next` opens Phase 0 and runs its steps up to the first gate — the normal flow resumes. +- **Testing the clock:** every read/advance command accepts `--as-of YYYY-MM-DD` to simulate a date (dry-run only). Real runs use today. + +**Changing the CCD (real production change).** If the user wants to move the date ("give us more time", "cut early"), use `set-ccd`. This **writes the pipeline override** — so it's gated: run it **without `--confirm` first to show the preview**, present that to the user, get an explicit yes (a `--reason` is always required, for audit), then re-run **with `--confirm`**. The override is month-scoped — the date must be in the release month. Use `--default` to revert to the 2nd-Wednesday default. + +**Skipping/cancelling the release.** Same gated pattern: `skip-release` sets the pipeline `skipRelease` switch (preview → confirm, reason required); `skip-release --clear` re-enables it. This suppresses the monthly trigger — treat it as a real, deliberate action and confirm before `--confirm`. + +**Ongoing conflict detection.** `status`/`resume` re-read the pipeline; if someone sets a differing override later, the same `ccd_conflict` surfaces — ask again. (Use `--no-pipeline-check` only if offline.) + +## Push reminders — the daily phase digest (reaching the user when Scout is closed) + +Everything above is **pull** (seen only when the user opens Scout). The **push** layer is a **daily phase status digest** emailed to the release owner, with a deliberate model: + +- **Setup is interactive — no push.** The readiness checklist and establishing the CCD happen live in Scout, so they are **never** emailed. An unsigned release, a blocked entry gate, and a halted release all stay silent. +- **The first push is a phase opening.** Phase 0 opens at **CCD‑7** — that's the first email. Nothing is sent before a phase opens (no pre‑open heads‑up). +- **Daily while a phase has outstanding work.** Once a phase is open, the owner gets a **once‑per‑day** digest (progress + what still needs them) until the phase's actions are done; then the next phase's digest takes over when it opens (each phase notifies on open). + +`tick` is the deterministic automation half: `python -m orchestrator.cli tick --json` first **advances** the active release (runs the agent steps that can run, holding at gates/actions — idempotent), then returns `{message, html, subject, owner_email, owner_name, release}` — `message` is the plain-text digest, `html` is the rich HTML version (full task table with status pills, attention-flagged), both empty when nothing is due today or it was already sent today. (`notify --json` is the read-only variant — same payload but does NOT advance; use it for a manual "what would I be told" check.) `--as-of ` is a debug clock; `--force` bypasses the once‑per‑day guard. + +- A **Scout automation** named **"Release push reminders"** runs `tick --json` (discovery mode) **hourly** and, when `message` is non‑empty, emails it via `workiq_send_email` to `owner_email` (subject from the JSON) — the release owner from release metadata, **never a hardcoded address**; when `message` is empty it stays silent. Running hourly (not once/day) means a tick missed while the machine was off is picked up by the next one, and idempotency + once‑per‑day de‑dup keep it to one advance-effect and one email per day. It is **per‑release**: auto‑provisioned (create‑if‑missing, registered to the release) at start and torn down at close. (Email is the channel because it reliably notifies; the `m_send_teams_message` bot relay 404s without a conversation reference, and the Teams self‑chat delivers silently.) + +If the user asks "how will I be reminded" / "set up notifications," explain this; if the automation doesn't exist, create it (see "Ensure push reminders exist"). Keep the email subject/body exactly as `tick` returns — don't embellish. + +## The readiness ENTRY GATE (right after starting) + +Immediately after `init`, the very first thing is the **readiness checklist** — the entry gate. The engine's `next` refuses to run any step (reports `readiness_gate`) until it's cleared. **Every item is equally required** — there is no priority or "hard vs soft" distinction. The only difference between items is **who resolves them**: + +- **`auto`** — **Scout resolves it** (verifies programmatically, pass/fail). Two execution sources, but the user sees both as `[auto]`: + - *Python-verified* (default): `build_access` (both ADO build definitions, via `az`) and `mcp_servers` (the ICM + Kusto/ADX MCP servers are registered in Scout). The engine's `verify`/`sign` runs these. + - *Scout-assisted* (`source: scout`): the **engine can't reach the MCP/Scout-settings, so YOU run the check** and record the result (see step 3a). Fail-closed **except `silent_perms`** (see below). Today: `oncall_now` (ICM current on-call), `adx_access` (Kusto `print 1` against the ADX cluster), `silent_perms` (Scout permissions allow fully-unattended runs). +- **`attest`** — **the engineer resolves it** (confirms): `play_console_access`, `oncall_window`, `saw_ame`, `yubikey`. + +**Two of the auto items exist so scheduled work runs UNATTENDED** (machine on, Scout not focused): `mcp_servers` (the MCP deps are registered — **hard**, since without them the on-call/telemetry checks can't run) and `silent_perms` (permissions won't stall the daily digest / Teams reminders / browser checks on a prompt — **soft/opt-out**: the user can choose to proceed without silent runs, recorded as `degraded`, with the downside noted). Enabling silent runs needs the user to flip ONE Scout master toggle first (*"Allow AI to request permission changes"*) — only they can (it's read-only from the model); after that I auto-request the rest with a single Allow click. + +**On-call is TWO items (hybrid), because Scout can only see the *current* rotation, not the future one:** +- `oncall_now` (**auto/ICM**) — are you on-call *right now*? Scout verifies this from ICM. +- `oncall_window` (**attest**) — are you free across the whole release window **CCD‑7 → CCD+14**? Scout can't read the future rotation, so you attest it (the checklist shows the concrete dates). + +If any item is unsatisfied the gate stays closed. If the engineer can't satisfy an attest item, they resolve it or hand the release to someone who can — the same for every item. Never describe any item as "not a hard block" or "optional." + +Flow after starting: +1. `python -m orchestrator.cli checklist --release --verify` — this runs the auto checks AND prints the **canonical checklist table (markdown)**. **Reproduce its stdout into your reply as live markdown (NOT wrapped in a ``` code fence)** so Scout renders it as a real table — it is already a finished markdown table with the type labels, per-item status, and clickable links. **Do NOT rebuild, re-format, re-order, re-label, or re-type any of it from memory, and do NOT fence it.** If you reconstruct it you WILL introduce errors (stale icons, mangled/merged URLs); if you fence it, it shows as raw text. Always reproduce the literal command output as rendered markdown. You may add a sentence of your own before or after, but the table block itself must match the output. +2. There are exactly **two types by resolver**: `[auto]` (Scout verifies) and `[attest]` (the user confirms). All items must be satisfied to clear the gate. (Do not add lock icons or a "hard requirement" legend.) +3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. + - **`oncall_now` (ICM):** call the ICM MCP `get_on_call_schedule_by_team_id` with `teamIds: [78848]` ("Auth Client Android Shield"). Resolve the current user's alias (`get_my_icm_context` or the owner email's local part), then decide by their role in `shiftCurrentOnCalls[].currentOnCallContacts[]`: + - **Not in the roster at all** → `record-check --item oncall_now --status pass --detail "not on the current roster"`. + - **Present but NOT the primary** (i.e. they are a **backup/secondary** — any position other than the first-listed contact) → **pass**: `record-check --item oncall_now --status pass --detail "backup OCE, not primary (primary: )"`. A backup is free to run the release. + - **The PRIMARY / current OCE** (the **first-listed** contact in `currentOnCallContacts`) → `record-check --item oncall_now --status fail --detail "currently the primary on-call for Auth Client Android Shield"`. + - Only the **primary** blocks the gate. If you cannot confidently tell primary from backup (ambiguous ordering, or the user says otherwise), **ask the user** "Are you the primary/current OCE, or backup?" and record accordingly — **never block a backup.** + - **`adx_access` (Kusto):** run a trivial query — `kusto_query` with the item's `cluster_uri` + `database` (from `checklist --json`) and query `print 1`. Success = the engineer has data access to the ADX release dashboard's cluster. + - Query succeeds → `record-check --release --item adx_access --status pass --detail "print 1 succeeded"`. + - Query fails (auth/access error) → `record-check --release --item adx_access --status fail --detail ""`. + - **`silent_perms` (Scout settings — OPT-OUT/soft):** the daily push digest, the Teams reminders and the browser (CCOA/lockdown) checks all run from a background automation while Scout isn't focused — they must not stall on a permission prompt. Call **`m_get_settings`** and read `permissions.servers`. It's satisfied when ALL of the item's `required_servers` (from `checklist --json`: `shell`, `workiq`, `playwright`) have `autoApprove: true` (one server flag each keeps it simple: `workiq.autoApprove` covers both `workiq_send_email` and Teams; `playwright.autoApprove` covers the browser). **This item never hard-blocks — the user may choose to proceed without silent runs.** Flow: + - **Already all auto-approved** → `record-check --release --item silent_perms --status pass --detail "shell/workiq/playwright auto-approved"`. Done. + - **One or more NOT auto-approved** → **offer the choice** with `m_ask_user`: **"Enable silent runs (recommended)"** vs **"Proceed without — I'll get prompts"**. Explain the downside of proceeding: *the daily digest, Teams reminders, and CCOA/lockdown browser checks will pop a permission prompt when Scout isn't focused and can stall until you open Scout and approve them.* + - They pick **Enable** → the only manual step is the Scout master toggle: if `permissions.allowModelPermissionsChange` is `false`, tell them to turn on **Settings → Permissions → "Allow AI to request permission changes"** (I cannot flip it — it's read-only from the model, by design). Once it's `true`, call **`m_request_permission_escalation`** with `servers: { workiq: {autoApprove:true}, playwright: {autoApprove:true} }` (add `shell` if off too); they click **Allow** once, then re-read `m_get_settings` and `record-check … --status pass --detail "enabled silent runs"`. + - They pick **Proceed without** (or won't enable the master toggle) → `record-check --release --item silent_perms --status degraded --detail "proceeding without silent runs — unattended digest/Teams/browser checks will prompt & may stall until Scout is opened"`. **`degraded` satisfies the gate** (the checklist shows it as ⚠️ *Proceeding (not silent)*), so the release can start; the downside is on record. + - A `fail` on `oncall_now`/`adx_access`, or a real problem, keeps the gate closed — treat it like any unsatisfiable required item (resolve or hand off). Do NOT attest these — they're `auto` items you verified. (`silent_perms` is the one soft/opt-out auto item: it uses `degraded`, never `fail`, when the user chooses to proceed.) +3b. Use `m_ask_user` to collect the **attestations**: `play_console_access`, the on-call **window** (`oncall_window` — show the CCD‑7 → CCD+14 dates from the checklist), `saw_ame`, `yubikey`. Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. +4. If they confirm everything → `sign --release --all`, then `next` to begin Phase 0. +5. **If they can't satisfy any attest item** (on-call during the window, no SAW, no YubiKey, no portal access) → `decline --release --item ` (repeat `--item` for each). The gate is now blocked. Tell them plainly: the release can't start until that item is resolved; if they can't resolve it, hand the release to another engineer who can (notify their manager / the release team). Treat every item this way — don't single any out as harder or softer. +6. If an **auto** item shows FAIL (no build-definition access, or you're on-call), the gate stays closed — a real problem to resolve, not something to attest around. + +Never attest an `auto` item on the user's behalf — auto items are only satisfied by real verification (Python check or your recorded ICM result). +Never hand-edit or regenerate the checklist/status blocks — always show the CLI's literal output. + +## Parallel phases — process ALL the holds, not one at a time + +Some phases run **in parallel** (Phase 0 is `execution: parallel`): a single `next` runs **every independent automated step at once** (breaking, CG, cron, wiki — all complete in one call) and then surfaces **all the human/scout holds together** (e.g. *"4 item(s) need you: …"*). So don't treat it as one-step-at-a-time. After `next`, read `status --json` and look at **`pending_human`** (and `active_phase.steps` with their `status`/`needs_owner`) — that's the full set of what's outstanding. Work through **all** of them in this pass: +- **`source: scout`** steps (notice, flight_reminder, lockdown) → run each via MCP/browser + `record-step` (see the sections below). These are independent — do them all. +- **`attest`** steps (confirm_reminders, vitals) → ask the owner to confirm, then `done --step `. +- **`blocked`** steps (cg/cron on a real problem) → show the note; fix + rerun, or skip. +Dependencies still hold: `confirm_reminders` only appears **after** `flight_reminder` is sent (it won't be in `pending_human` until then). Call `next` again after clearing holds to let newly-ready steps surface and, once all are done, advance to the next phase. + +## Scout-assisted phase steps (CCOA lockdown check) + +Some Phase steps read AAD-gated sources the deterministic engine can't reach, so **you run them via the browser and record the result** — same idea as the readiness scout checks, but mid-phase. When advancing, if **`lockdown`** is among the pending holds (`source: scout`, in `pending_human`), handle it like this — silently, without bothering the user unless there's an overlap: + +1. **Scrape the CCOA source.** Navigate (Playwright) to `https://prod.change-manager.msidentity.com/ccoa-periods`. If an AAD account picker appears, click the user's own account (Windows-SSO — no password). Wait for the "CCOA Periods" page. +2. **Extract the periods.** From **"Upcoming CCOA periods"** and the **current-year** "Past NoFly Zones" table, read each row's **Name, Environment, Start Date (UTC), End Date (UTC)**. Build a JSON array: `[{"name","environment","start":"YYYY-MM-DD","end":"YYYY-MM-DD"}, ...]` (use the UTC dates). +3. **Let the engine decide (deterministic).** Run `python -m orchestrator.cli check-lockdown --release --periods-json ''`. It computes the release window (CCD‑7 … CCD+14), keeps only **Production**-environment periods, checks overlap, and records the step: **pass** (no overlap → step done, flow continues) or **attention** (overlap → step holds). +4. **Relay the outcome.** On **pass**, just continue (`next`) — no need to bother the user. On **attention**, surface it: name the overlapping lockdown(s) and window, and tell them to **shift CCD** past the lockdown (`set-ccd`) if they want to proceed; there are **no partners to notify** for this step. + +If you can't reach the browser/SSO in this context, leave the step held — it stays flagged as needing attention and you (or the user, next time Scout is open) can run it then. Don't mark it done without actually running the check. + +## Scout-assisted phase steps (early code-complete notice) + +The Phase-0 `notice` step sends the early code-complete email. Sending needs WorkIQ (a skill capability), so it's scout-assisted like `lockdown`. When `status --json` shows the current step is **`notice`** (holding, `awaiting_action`): + +1. **Prepare it (deterministic).** Run `python -m orchestrator.cli prepare-notice --release `. It fills the local template (`templates/early-code-complete-notice.md`) with the release's CCD/owner and returns JSON `{subject, body, html, recipients, dry_run, recipients_note}`. +2. **Send it.** Email via `workiq_send_email` using the returned `subject` and `recipients` exactly, with **`body:` the `html` value and `isHtml: true`** (the HTML has a clean hotfix-guide link + a proper rendered table — fall back to the plain `body` with `isHtml: false` only if `html` is empty). **Recipients are already resolved for you**: in a **dry-run** they're the **release owner only** (safe rehearsal — the subject is prefixed `[DRY-RUN → owner]`); on a **live** release they're the real distribution list (androididentity@microsoft.com, jialh@microsoft.com — see EXTERNAL-REFERENCES.md). Never override the recipients. +3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step notice --status pass --detail "sent to "`. If the send fails, `--status attention --detail ""` to keep it flagged. + +## Scout-assisted phase steps (flight & string reminders — Teams) + +The Phase-0 `flight_reminder` step posts a **combined 4-in-1 reminder** (update local flights · flight pre-mortem docs · merge user-facing strings by CCD-7 · Auth App feature-flag freeze / default-OFF review) as a **Teams message** to the Android Core Team. Sending Teams needs WorkIQ, so it's scout-assisted. When the current step is **`flight_reminder`** (holding): + +1. **Prepare it.** Run `python -m orchestrator.cli prepare-flight-reminder --release `. It returns JSON `{content, content_type:"html", dry_run, send_to, owner_email, chat_id, target_note}`. +2. **Resolve the chat + send.** + - **Dry-run** (`send_to: "owner"`): get the owner's 1:1 chat with `workiq_create_chat_by_email` (email = `owner_email`), then `workiq_send_chat_message` with that `chatId`, `content` = the returned HTML, `contentType: "html"`. (Safe rehearsal — the message is prefixed `[DRY-RUN → owner]`.) + - **Live** (`send_to: "group"`): `workiq_send_chat_message` with `chatId` = the returned `chat_id` (the Android Core Team thread), `content`, `contentType: "html"`. + Never override the target — `prepare-flight-reminder` already picked owner-vs-group from dry_run. +3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step flight_reminder --status pass --detail "posted to "`; on failure, `--status attention --detail ""`. + +**Sending the reminder is fire-and-forget** — it does NOT prove the feature owners actually did the work. So the very next step is **`confirm_reminders`**, a human **attestation** the engine holds on (`awaiting_action`). When the current step is `confirm_reminders`, ask the release owner (via `m_ask_user`) to confirm the reminded work is actually done — feature owners updated local flights, wrote flight pre-mortem docs, merged user-facing strings by CCD-7, and all features are default-OFF (or default-ON ones are approved in the wiki). Only when they confirm, run `python -m orchestrator.cli done --release --step confirm_reminders --note ""`. If they can't confirm, leave it holding (the release correctly blocks here until the pre-requisite work is verified) — don't mark it done. + +Phase 0's **`vitals`** step ("Confirm Play Console vitals & policy status reviewed") is another **attestation** hold. Play Console has no API for **Policy issues/warnings** (the Reporting API covers only technical vitals, and the Console UI is behind a Google login Scout can't automate), so this is a manual check: when the current step is `vitals`, ask the owner to open Play Console, review **Android vitals** (crash/ANR rate) and **Policy status** (issues/warnings), and confirm they're acceptable. On confirmation, `python -m orchestrator.cli done --release --step vitals --note ""`. If there's an unresolved policy issue or vitals regression, leave it holding. + +## Commands (run from the release-agent folder) + +| Intent | Command | +| --- | --- | +| Discover releases | `python -m orchestrator.cli list --json` | +| Start a new release (dry-run) | `python -m orchestrator.cli init --release ` | +| Start for real | `python -m orchestrator.cli init --release --live` | +| Show readiness entry checklist | `python -m orchestrator.cli checklist --release --json` | +| Run auto readiness verifiers | `python -m orchestrator.cli verify --release ` | +| Attest human items (+auto verify) | `python -m orchestrator.cli sign --release --all` | +| Record a scout-assisted check (ICM on-call now) | `python -m orchestrator.cli record-check --release --item oncall_now --status pass\|fail --detail "..."` | +| Decide CCOA lockdown overlap (from scraped periods) | `python -m orchestrator.cli check-lockdown --release --periods-json '[{"name","environment","start","end"}]'` | +| Prepare the early code-complete notice email (JSON) | `python -m orchestrator.cli prepare-notice --release [--variant initial\|update]` | +| Prepare the flight & string reminders Teams message (JSON) | `python -m orchestrator.cli prepare-flight-reminder --release ` | +| Record a scout-assisted phase step (after sending/doing it) | `python -m orchestrator.cli record-step --release --step --status pass\|attention --detail "..."` | +| Declare you CANNOT satisfy an item | `python -m orchestrator.cli decline --release --item ` | +| Status (structured) | `python -m orchestrator.cli status --release --json` | +| Advance to next gate | `python -m orchestrator.cli next --release ` | +| Approve the holding gate | `python -m orchestrator.cli approve --release --comment ""` | +| Deny the holding gate | `python -m orchestrator.cli deny --release --comment ""` | +| **Done** — mark a reminder (human to-do) complete | `python -m orchestrator.cli done --release [--phase

--step ] --note ""` | +| **Set/change CCD** (writes pipeline; preview→confirm) | `python -m orchestrator.cli set-ccd --release --date --reason "" [--confirm]` | +| **Revert CCD to default** (2nd Wednesday) | `python -m orchestrator.cli set-ccd --release --default --reason "" [--confirm]` | +| **Skip/cancel the release** (writes pipeline) | `python -m orchestrator.cli skip-release --release --reason "" [--confirm]` (add `--clear` to un-skip) | +| **Skip** a step (reason REQUIRED) | `python -m orchestrator.cli skip --release --phase

--step --reason ""` | +| **Reopen** a done/skipped step | `python -m orchestrator.cli reopen --release --phase

--step [--reason "..."]` | +| **Halt** (emergency, reason REQUIRED) | `python -m orchestrator.cli halt --release --reason ""` | +| **Resume** after a halt | `python -m orchestrator.cli resume --release [--reason "..."]` | +| Show / analyze this release's log | `python -m orchestrator.cli log --release ` (add `--analyze`, `--json`) | +| Journal interaction (silent) | `python -m orchestrator.cli journal --release --source scout|user --text "..."` | +| Activate conditional hotfix phase | `python -m orchestrator.cli activate --release --phase hotfix` | +| **Notify** — push line if something needs me (else nothing) | `python -m orchestrator.cli notify [--release ] [--as-of ] [--force]` | +| **Track automations** (register/list/deregister for teardown) | `python -m orchestrator.cli automation register --id --name "" [--shared\|--release ] [--purpose "..."]` · `automation list [--release ] [--json]` · `automation deregister --id ` | + +**Manual overrides** (the release engineer can steer when reality diverges from the plan): +- **skip** — a step doesn't apply this release, or was done manually outside the tool. **A reason is required** (audited). Confirm the reason with the user, then run `skip`. +- **reopen** — a step (incl. an approved gate) needs to run again; reopening a gate makes it re-hold for a fresh decision. +- **halt** — emergency freeze (e.g. production incident). **Reason required.** While halted, `next` refuses to advance and status shows a HALTED banner. Use for "stop everything now." +- **resume** — clear a halt and continue. +Map natural language to these ("skip the CG report, doesn't apply" → `skip … --reason`; "halt, we have an incident" → `halt --reason`; "resume" → `resume`). Never skip or halt without capturing the user's reason. + +The human-readable commands (`checklist`, `status`, `next`, `approve`, `deny`, `decline` without `--json`) emit a **canonical block AND auto-log it** as scout output. **Prefer these and show their output to the user** — that keeps the display consistent AND guarantees the log captures what was shown. Use `--json` only when you need raw fields for your own logic, not for display. + +## Presenting STATUS (make it clean — users ask for this most) + +> **Render it as markdown, never as a code block.** Both the `checklist` and `status` outputs are already markdown (headings + tables). Paste them into your reply as **normal message content so Scout renders the table/stepper** — do **NOT** wrap them in a ``` code fence / triple backticks. Fencing them makes them show as raw text (YAML-looking) instead of a rendered table. Reproduce the content faithfully, but as live markdown. + +**First decide what to show.** If the readiness entry gate isn't cleared yet (not signed, or blocked), the most useful "status" is the **checklist itself** — run `checklist --release --verify` and show that table (don't show the terse status line, and don't ask permission to pull the checklist). Only when the gate is cleared / the release is mid-flight do you show the `status` block. + +For a mid-release status: run `python -m orchestrator.cli status --release ` (no `--json`) and **show its output as rendered markdown** (not fenced) — it's a finished view with a next-action headline, a **phase map** (✅ done · ⏸ in progress · 🗓 scheduled · ⬜ not started) and the **current phase's steps** in a table. It auto-logs what was shown. You may add a sentence before/after, but reproduce the block faithfully; don't re-render from `--json` (that skips the auto-log) and don't invent your own layout. Never surface raw engine state names like `holding_gate`, `awaiting_action`, or `scheduled` — the view already translates them to plain language ("Waiting for your approval", "Action needed from you", "Scheduled"). + +If you need structured fields for branching logic, `status --json` gives: +`release_id, status, dry_run, ccd, ccd_source, as_of, done, total, percent, current_phase_name, current_step_name, gate, action, scheduled, pending_human, readiness_signed, blocked`. + +## Behaviour +1. **"status" / "where are we":** discover the release, then check its state: + - **If the readiness entry gate is not yet cleared** (status `readiness_gate` / not signed, or `blocked`): the useful answer *is* the checklist — so run `checklist --release --verify` and **show that table directly**. Don't show the terse "entry gate not signed" line and don't ask "want me to pull the checklist?" — just pull it. Then prompt for the attestations (see the ENTRY GATE flow). + - **Otherwise** (gate cleared / mid-release): show the `status` output. + - **If no release exists:** say so briefly and use `m_ask_user` to offer starting one. +2. **"start a release":** use `m_ask_user` to offer current-month vs another month (you compute the `YYYY-MM`), run `init`, **ensure the push-reminder automation exists** (create-if-missing — see "Ensure push reminders exist"), then **present the readiness entry checklist and get it signed** (see "The readiness ENTRY GATE"), then `next`, then present the resulting status. +3. **Engine HOLDS at a gate:** stop and present the gate. Use `m_ask_user` to offer **Approve** / **Deny** (never decide for them). +4. **On their decision:** run `approve`/`deny` with their comment, then present the new status (it auto-continues to the next gate). +5. **Engine HOLDS for a reminder (ACTION NEEDED / status `awaiting_action`):** this is a human *to‑do*, not a decision — the engine can't do it and is waiting for the person to do it (e.g. "China publish flow", "Surface Phase 2 UI failure list"). Present it plainly as "you need to do X", and when the user says it's done, run `done --release --note ""` (defaults to the current held step) — the flow then advances. Don't offer Approve/Deny for a reminder; it's just done / not-yet. +6. **A step is BLOCKED (an agent found a real problem):** some agent steps block the flow when they detect something the owner must resolve. Today: **`cg`** (blocks on active **High/Critical** Component Governance alerts — the note lists CVE, component, fix) and **`cron`** (blocks if the Calendar Checker pipeline has no recent scheduled run — i.e. the cron may be broken). Show the owner the note plainly. Their two exits for any blocked step: **(a) fix** the underlying problem, then **rerun** — call `next` again, which re-runs the check; if it's now clean it passes and the flow continues; **(b) override** — `skip --release --phase --step --reason ""` (e.g. accepted risk, tracked separately). Don't mark it done any other way — either the re-check passes or they consciously skip. +7. **Engine is `scheduled`:** Phase 0 hasn't opened yet (before CCD‑7). Tell the user the opens date + countdown from the status view; there's nothing to advance. If they want to start earlier, that's a CCD change (`set-ccd`), not a `next`. +8. **"continue"/"resume":** discover → if the entry gate isn't cleared, show the checklist (as in #1); otherwise brief them with status → then `next`. + +> **Prompt, don't interrogate.** Whenever you need a discrete choice from the user (start? which release? approve or deny?), prefer the `m_ask_user` clickable prompt over a free-text question. Reserve free-text prompts for genuinely open values (like an unusual month). + +## Guardrails +- The engine is the source of truth for sequencing and gate state. When unsure, run `status --json`. +- Gates are **human-decided**. Present and relay; never authorize. +- **Dry-run by default.** Only use `--live` when the user explicitly asks. +- `[STUB...]` output means that step is mocked today — say so plainly; don't imply real work happened. + +## Event logging (silent — never changes the interaction) +Each release keeps its **own** append-only log at `.release-runs//events.jsonl` (per-release only — there is no machine-wide aggregate). It exists purely for debugging and improvement. **Logging is invisible to the user and must never change how you communicate** — do not announce it, and do not add extra questions just to populate it. + +**What's automatic vs. your job:** +- **Scout output is logged automatically** by the CLI whenever you run a human-readable command (`checklist`, `status`, `next`, `approve`, `deny`, `decline` without `--json`). You do NOT need to journal what was shown — just run those commands and show their output. +- **User input is your responsibility** — the engine can't see what the user typed/clicked in Scout. So **every time the user makes a choice or gives input, immediately journal it** (this is required, not optional): + `journal --release --source user --kind choice --text "" --choice "

  • ") == 4 + assert "variableGroupId=40" in out["content"] # variable-group link + + # live + st2 = ReleaseState(release_id=rid, dry_run=False, ccd="2026-08-12", + ccd_source="default", owner_email="pedroro@microsoft.com") + C.save_state(st2, d, rid) + buf2 = io.StringIO() + with contextlib.redirect_stdout(buf2): + ncmd.cmd_prepare_flight_reminder(A) + out2 = _json.loads(buf2.getvalue()) + assert out2["dry_run"] is False and out2["send_to"] == "group" + assert out2["chat_id"] == "19:976a859f167f44e59c4ceca8b1d23581@thread.v2" + assert "[DRY-RUN" not in out2["content"] + + +def test_no_localization_strings_step(): + """The old #5 localization strings step was removed.""" + st, orch = _orch() + preflight = next(p for p in orch.config["phases"] if p["id"] == "preflight") + assert "strings" not in [s["id"] for s in preflight["steps"]] + + +def test_confirm_reminders_is_attestation_hold(): + """In the parallel Phase 0, confirm_reminders is a human attestation that becomes + ready only after flight_reminder is sent; it surfaces as a hold with a confirm + pill and clears via `done`.""" + st, orch = _orch(signed=False) + _pass_scout_checks(orch) + orch.gate.sign() + _clear_notice(orch) + orch.record_scout_step("preflight", "flight_reminder", "pass", "sent") + orch.run_until_gate() + assert st.status == "awaiting_action" + # confirm_reminders (dep on flight_reminder, now done) is among the pending holds + assert "preflight.confirm_reminders" in st.pending_human + ap = orch.status_report()["active_phase"] + step = next(s for s in ap["steps"] if s["id"] == "confirm_reminders") + assert step["status"] == "confirm" and step["needs_owner"] + # owner attests → advances + orch.complete_step("preflight", "confirm_reminders", "verified with feature owners") + assert st.is_done("preflight", "confirm_reminders") + + +def test_confirm_reminders_gated_by_flight_send(): + """confirm_reminders must NOT be offered until flight_reminder is sent (dependency).""" + st, orch = _orch(signed=False) + _pass_scout_checks(orch) + orch.gate.sign() + _clear_notice(orch) + # flight_reminder NOT yet recorded → confirm_reminders is not ready + orch.run_until_gate() + assert "preflight.confirm_reminders" not in st.pending_human + assert "preflight.flight_reminder" in st.pending_human # the send is what's pending + + +def test_parallel_autos_run_despite_pending_holds(): + """Independent auto steps (breaking/cg/cron/wiki) complete even while scout/attest + steps are still holding — a hold no longer blocks its siblings.""" + st, orch = _orch(signed=False) + _pass_scout_checks(orch) + orch.gate.sign() + # nothing cleared: notice/flight/lockdown (scout) + vitals (attest) all hold + orch.run_until_gate() + # yet the independent auto agents ran to completion + for sid in ("breaking", "cg", "cron", "wiki"): + assert st.is_done("preflight", sid), sid + # and the holds are all surfaced together + for sid in ("notice", "flight_reminder", "lockdown", "vitals"): + assert f"preflight.{sid}" in st.pending_human, sid + + +def test_cg_report_summarizes_and_flags_high(): + """The CG report groups active alerts by severity and lists High/Critical.""" + from phases.agents.preflight import _cg_summary, _cg_report + alerts = [ + {"alertState": "active", "severity": "high", "title": "CVE-1", + "component": {"displayName": "io.netty:x", "displayVersion": "4.2.15"}, + "actionItems": "Upgrade to 4.2.16"}, + {"alertState": "active", "severity": "medium", "title": "CVE-2"}, + {"alertState": "autoDismissed", "severity": "high", "title": "CVE-OLD"}, + {"alertState": "fixed", "severity": "critical", "title": "CVE-FIXED"}, + ] + active, high = _cg_summary(alerts, ["critical", "high"]) + assert len(active) == 2 and len(high) == 1 + rep = _cg_report(active, high) + assert "2 active alert(s)" in rep and "1 high" in rep + assert "CVE-1" in rep and "io.netty:x" in rep and "Upgrade to 4.2.16" in rep + assert "CVE-OLD" not in rep and "CVE-FIXED" not in rep # non-active excluded + + +def test_cg_agent_dry_run_simulates(): + from phases.agents import preflight as pa + r = pa.run_cg_alerts("preflight", {"id": "cg"}, True, None) + assert r.ok and "dry-run" in r.action.lower() + + +def test_cg_agent_blocks_on_high(): + """High/Critical active alerts BLOCK the step (ok=False) with a fix-and-rerun message.""" + from phases.agents import preflight as pa + from tools import checks + orig = checks.fetch_cg_alerts + checks.fetch_cg_alerts = lambda *a, **k: (True, [ + {"alertState": "active", "severity": "high", "title": "CVE-9", + "component": {"displayName": "pkg", "displayVersion": "1.0"}}, + ], "ok") + try: + r = pa.run_cg_alerts("preflight", {"id": "cg"}, False, None) + assert not r.ok # High → blocks + assert "CVE-9" in r.action and "RERUN" in r.action + finally: + checks.fetch_cg_alerts = orig + + +def test_cg_agent_passes_when_no_high(): + """Only Medium/Low active alerts → the step passes (report captured).""" + from phases.agents import preflight as pa + from tools import checks + orig = checks.fetch_cg_alerts + checks.fetch_cg_alerts = lambda *a, **k: (True, [ + {"alertState": "active", "severity": "medium", "title": "CVE-M"}, + ], "ok") + try: + r = pa.run_cg_alerts("preflight", {"id": "cg"}, False, None) + assert r.ok and "1 active" in r.action + finally: + checks.fetch_cg_alerts = orig + + +def test_cg_blocked_step_reruns_and_clears_when_fixed(): + """A CG block holds the step; fixing (alerts now clean) + rerunning `next` + re-checks and lets the flow continue.""" + from phases import agents as pa + from phases.stub_runner import StepResult + flag = {"high": True} + + def fake_cg(phase, step, dry_run, st): + if flag["high"]: + return StepResult(False, "CG: 1 critical active\n→ Fix and RERUN or skip.", "agent") + return StepResult(True, "CG: 0 active alerts.", "agent") + orig = pa.REGISTRY["cg_alerts"] + pa.REGISTRY["cg_alerts"] = fake_cg + try: + st, orch = _orch() + _clear_phase0_scout(orch) # clear the earlier scout/human holds + orch.run_until_gate() + assert st.status == "awaiting_action" and st.current_step == "cg" + assert st.get_step("preflight", "cg").status == "blocked" + # the digest shows it blocked / needs owner + step = next(s for s in orch.status_report()["active_phase"]["steps"] if s["id"] == "cg") + assert step["status"] == "blocked" and step["needs_owner"] + # FIX: alerts now clean → RERUN (next) re-checks and passes + flag["high"] = False + orch.run_until_gate() + assert st.is_done("preflight", "cg") + finally: + pa.REGISTRY["cg_alerts"] = orig + + +def test_cg_blocked_step_skip_override(): + """The owner can override a CG block by skipping the step (with a reason).""" + from phases import agents as pa + from phases.stub_runner import StepResult + orig = pa.REGISTRY["cg_alerts"] + pa.REGISTRY["cg_alerts"] = lambda *a, **k: StepResult(False, "CG: 1 high active", "agent") + try: + st, orch = _orch() + _clear_phase0_scout(orch) + orch.run_until_gate() + assert st.current_step == "cg" and st.get_step("preflight", "cg").status == "blocked" + orch.skip_step("preflight", "cg", "accepted risk; tracked separately") + assert st.is_done("preflight", "cg") # skipped counts as done + assert st.get_step("preflight", "cg").status == "skipped" + finally: + pa.REGISTRY["cg_alerts"] = orig + + +def test_cg_agent_fetch_error_holds(): + from phases.agents import preflight as pa + from tools import checks + orig = checks.fetch_cg_alerts + checks.fetch_cg_alerts = lambda *a, **k: (False, [], "403 forbidden") + try: + r = pa.run_cg_alerts("preflight", {"id": "cg"}, False, None) + assert not r.ok and "could not read alerts" in r.action + finally: + checks.fetch_cg_alerts = orig + + +def test_cron_check_dry_run_simulates(): + from phases.agents import preflight as pa + r = pa.run_cron_check("preflight", {"id": "cron"}, True, None) + assert r.ok and "dry-run" in r.action.lower() + + +def test_cron_check_passes_on_recent_scheduled_run(): + from phases.agents import preflight as pa + from tools import checks + from datetime import datetime, timezone + orig = checks.latest_scheduled_build + now_iso = datetime.now(timezone.utc).isoformat() + checks.latest_scheduled_build = lambda *a, **k: (True, { + "queueTime": now_iso, "result": "succeeded", "status": "completed"}, "ok") + try: + r = pa.run_cron_check("preflight", {"id": "cron"}, False, None) + assert r.ok and "scheduled and firing" in r.action + finally: + checks.latest_scheduled_build = orig + + +def test_cron_check_blocks_when_stale(): + from phases.agents import preflight as pa + from tools import checks + orig = checks.latest_scheduled_build + checks.latest_scheduled_build = lambda *a, **k: (True, { + "queueTime": "2026-01-01T06:00:00Z", "result": "succeeded", "status": "completed"}, "ok") + try: + r = pa.run_cron_check("preflight", {"id": "cron"}, False, None) + assert not r.ok and "stale" in r.action + finally: + checks.latest_scheduled_build = orig + + +def test_cron_check_blocks_when_no_scheduled_run(): + from phases.agents import preflight as pa + from tools import checks + orig = checks.latest_scheduled_build + checks.latest_scheduled_build = lambda *a, **k: (True, None, "no scheduled runs in recent history") + try: + r = pa.run_cron_check("preflight", {"id": "cron"}, False, None) + assert not r.ok and "no scheduled run" in r.action + finally: + checks.latest_scheduled_build = orig + + +def test_vitals_is_attestation_hold(): + """Play Console vitals/policy is an attest step (no API for policy): it HOLDS + for the owner to confirm they reviewed it, and clears via `done`.""" + st, orch = _orch() # clears the earlier holds (incl. vitals via _clear_phase0_scout) + # re-open vitals to observe its natural hold + orch.reopen_step("preflight", "vitals") + orch.run_until_gate() + assert st.status == "awaiting_action" + assert st.current_step == "vitals" + step = next(s for s in orch.status_report()["active_phase"]["steps"] if s["id"] == "vitals") + assert step["status"] == "confirm" and step["needs_owner"] + orch.complete_step("preflight", "vitals", "reviewed vitals + policy status in Play Console") + assert st.is_done("preflight", "vitals") + + +if __name__ == "__main__": + fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + for fn in fns: + fn() + print(f" PASS {fn.__name__}") + print(f"\n{len(fns)}/{len(fns)} tests passed") diff --git a/release-agent/tools/__init__.py b/release-agent/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release-agent/tools/checks.py b/release-agent/tools/checks.py new file mode 100644 index 00000000..677f09e7 --- /dev/null +++ b/release-agent/tools/checks.py @@ -0,0 +1,285 @@ +"""Real checks for readiness verifiers (no fakery). + +- ado_build_def: uses `az pipelines build definition show` to prove the signed-in + user can actually access a build definition. Genuine access check. +- http_reachable: HEAD/GET a URL and report the status. NOTE: for auth-gated web + apps (Play Console, ADX) an HTTP 200 only proves the URL is reachable — it does + NOT prove the user has sign-in access, since a login page also returns 200. +""" +from __future__ import annotations +import subprocess +import shutil +from dataclasses import dataclass +from urllib import request as _request +from urllib.error import URLError, HTTPError + + +@dataclass +class CheckResult: + ok: bool + verified_access: bool # True only when we truly proved access (not mere reachability) + detail: str + + +def check_ado_build_def(org: str, project: str, def_id: int, timeout: int = 30) -> CheckResult: + az = shutil.which("az") + if az is None: + return CheckResult(False, False, "az CLI not found") + try: + out = subprocess.run( + [az, "pipelines", "build", "definition", "show", + "--id", str(def_id), "--org", org, "--project", project, + "--query", "name", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, + ) + except subprocess.TimeoutExpired: + return CheckResult(False, False, f"timeout querying build def {def_id}") + except OSError as e: + return CheckResult(False, False, f"failed to run az: {e}") + if out.returncode == 0 and out.stdout.strip(): + return CheckResult(True, True, f"accessible: '{out.stdout.strip()}'") + err = (out.stderr or "").strip().splitlines() + msg = err[-1] if err else f"cannot access build def {def_id}" + return CheckResult(False, True, msg[:160]) + + +def current_az_user(timeout: int = 20): + """Return the signed-in user's email/UPN from `az account show`, or None. + Used to resolve the release owner without hardcoding an address.""" + az = shutil.which("az") + if az is None: + return None + try: + out = subprocess.run( + [az, "account", "show", "--query", "user.name", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if out.returncode == 0: + val = (out.stdout or "").strip() + return val or None + return None + + +def check_http(url: str, timeout: int = 15) -> CheckResult: + """Reachability check. verified_access is False by design — a 200 from an + auth-gated web app does not prove the user has access.""" + req = _request.Request(url, method="HEAD", headers={"User-Agent": "release-agent-readiness/1.0"}) + try: + with _request.urlopen(req, timeout=timeout) as resp: + code = resp.status + return CheckResult(200 <= code < 400, False, f"reachable (HTTP {code})") + except HTTPError as e: + # Some servers reject HEAD; treat <500 as reachable + return CheckResult(e.code < 500, False, f"reachable (HTTP {e.code})") + except (URLError, TimeoutError) as e: + return CheckResult(False, False, f"unreachable: {e}") + except Exception as e: # noqa + return CheckResult(False, False, f"error: {e}") + + +# ---- CCD pipeline variables (the source of record for Code Complete Date) ---- +# +# These read/write the definition-level variables on pipeline 3038 via the az CLI. +# `overrideCodeCompleteDate` and `skipRelease` are UI/definition variables, which +# is exactly what `az pipelines variable` operates on. + +def read_pipeline_variable(org: str, project: str, def_id: int, name: str, + timeout: int = 30): + """Return (ok, value, detail). value is the variable's string value (may be '') + or None if the variable isn't defined. ok is False only on a real access/CLI error.""" + az = shutil.which("az") + if az is None: + return (False, None, "az CLI not found") + try: + out = subprocess.run( + [az, "pipelines", "variable", "list", "--pipeline-id", str(def_id), + "--org", org, "--project", project, "-o", "json"], + capture_output=True, text=True, timeout=timeout, + ) + except subprocess.TimeoutExpired: + return (False, None, f"timeout listing variables on pipeline {def_id}") + except OSError as e: + return (False, None, f"failed to run az: {e}") + if out.returncode != 0: + err = (out.stderr or "").strip().splitlines() + return (False, None, (err[-1] if err else "az returned non-zero")[:160]) + import json as _json + try: + data = _json.loads(out.stdout or "{}") + except ValueError: + return (False, None, "could not parse az output") + if name in data: + return (True, (data[name] or {}).get("value", "") or "", "ok") + return (True, None, f"variable '{name}' not defined") + + +def set_pipeline_variable(org: str, project: str, def_id: int, name: str, + value: str, timeout: int = 30) -> CheckResult: + """Write a definition variable (update, creating it if absent). A real + production change — callers must gate this behind explicit confirmation.""" + az = shutil.which("az") + if az is None: + return CheckResult(False, False, "az CLI not found") + base = [az, "pipelines", "variable", "{verb}", "--pipeline-id", str(def_id), + "--org", org, "--project", project, "--name", name, "--value", value] + + def _run(verb): + cmd = [c.replace("{verb}", verb) for c in base] + try: + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + return None + except OSError: + return None + + out = _run("update") + if out is not None and out.returncode == 0: + return CheckResult(True, True, f"{name} = '{value}'") + # update fails if the variable doesn't exist yet — try create. + out2 = _run("create") + if out2 is not None and out2.returncode == 0: + return CheckResult(True, True, f"{name} created = '{value}'") + err = "" + for o in (out2, out): + if o is not None and (o.stderr or "").strip(): + err = (o.stderr or "").strip().splitlines()[-1] + break + return CheckResult(False, True, (err or f"could not set {name}")[:200]) + + +# ---- ADO wiki (payload subpage) -------------------------------------------- +# +# Creates a wiki page via `az devops wiki page create`. A real production write, +# so callers must gate it (the pre-flight agent only calls this on a non-dry-run +# release). Idempotent-ish: an already-existing page is treated as success. + +def wiki_page_exists(org: str, project: str, wiki: str, path: str, + timeout: int = 30): + """True/False whether a wiki page exists, or None if it can't be determined + (e.g. az missing / CLI error) — callers should treat None conservatively.""" + az = shutil.which("az") + if az is None: + return None + try: + out = subprocess.run( + [az, "devops", "wiki", "page", "show", "--path", path, + "--wiki", wiki, "--org", org, "--project", project, "-o", "json"], + capture_output=True, text=True, timeout=timeout, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if out.returncode == 0: + return True + if "could not be found" in (out.stderr or "").lower() or "notfound" in (out.stderr or "").lower(): + return False + return None + + +def create_wiki_page(org: str, project: str, wiki: str, path: str, + content: str, timeout: int = 60) -> CheckResult: + az = shutil.which("az") + if az is None: + return CheckResult(False, False, "az CLI not found") + import tempfile + import os as _os + fd, tmp = tempfile.mkstemp(suffix=".md") + _os.close(fd) + try: + with open(tmp, "w", encoding="utf-8") as fh: + fh.write(content) + cmd = [az, "devops", "wiki", "page", "create", "--path", path, + "--wiki", wiki, "--org", org, "--project", project, + "--file-path", tmp, "--output", "json"] + try: + out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + return CheckResult(False, True, f"timeout creating wiki page '{path}'") + except OSError as e: + return CheckResult(False, False, f"failed to run az: {e}") + if out.returncode == 0: + return CheckResult(True, True, f"created '{path}'") + stderr = (out.stderr or "").strip() + if "exist" in stderr.lower(): # already there — fine + return CheckResult(True, True, f"page '{path}' already exists") + msg = stderr.splitlines()[-1] if stderr else f"could not create '{path}'" + return CheckResult(False, True, msg[:200]) + finally: + try: + _os.remove(tmp) + except OSError: + pass + + +# ---- Component Governance alerts (read-only) ------------------------------- +# +# CG alerts live on a separate governance host and are read via `az rest` +# (the signed-in user's token). Read-only — we only report. + +def fetch_cg_alerts(resource: str, host: str, project_id: str, repo_id: int, + branch: str, timeout: int = 60): + """Return (ok, alerts, detail). `alerts` is the raw list of alert dicts for + the branch (all states). ok is False on a real CLI/access error.""" + az = shutil.which("az") + if az is None: + return (False, [], "az CLI not found") + url = (f"{host}/{project_id}/_apis/ComponentGovernance/GovernedRepositories/" + f"{repo_id}/Branches/{branch}/Alerts") + try: + out = subprocess.run( + [az, "rest", "--method", "get", "--resource", resource, "--uri", url, "-o", "json"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8", + ) + except subprocess.TimeoutExpired: + return (False, [], "timeout querying Component Governance alerts") + except OSError as e: + return (False, [], f"failed to run az: {e}") + if out.returncode != 0: + err = (out.stderr or "").strip().splitlines() + return (False, [], (err[-1] if err else "az returned non-zero")[:200]) + import json as _json + try: + data = _json.loads(out.stdout or "{}") + except ValueError: + return (False, [], "could not parse az output") + return (True, data.get("value", []) or [], "ok") + + +# ---- scheduled-pipeline verification (Calendar Checker) --------------------- +# +# A YAML pipeline's cron schedule isn't exposed in its definition triggers, but +# whether it's actually FIRING is provable from its build history: a recent +# `schedule`-reason run means the cron is live. Read-only. + +def latest_scheduled_build(org: str, project: str, def_id: int, timeout: int = 60): + """Return (ok, run, detail). `run` is a dict {queueTime, result, status} for + the most recent schedule-reason build, or None if none in recent history.""" + az = shutil.which("az") + if az is None: + return (False, None, "az CLI not found") + try: + out = subprocess.run( + [az, "pipelines", "build", "list", "--definition-ids", str(def_id), + "--org", org, "--project", project, "--top", "25", "-o", "json"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8", + ) + except subprocess.TimeoutExpired: + return (False, None, f"timeout listing builds for definition {def_id}") + except OSError as e: + return (False, None, f"failed to run az: {e}") + if out.returncode != 0: + err = (out.stderr or "").strip().splitlines() + return (False, None, (err[-1] if err else "az returned non-zero")[:200]) + import json as _json + try: + builds = _json.loads(out.stdout or "[]") + except ValueError: + return (False, None, "could not parse az output") + sched = [b for b in builds if b.get("reason") == "schedule"] + if not sched: + return (True, None, "no scheduled runs in recent history") + latest = max(sched, key=lambda b: b.get("queueTime") or "") + return (True, {"queueTime": latest.get("queueTime"), "result": latest.get("result"), + "status": latest.get("status")}, "ok") + diff --git a/settings.gradle b/settings.gradle index 97a748c6..63cdf70b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -186,23 +186,23 @@ project(':labapi').projectDir = new File('common/labapi') include(":keyvault") project(':keyvault').projectDir = new File('common/keyvault') -include(":AcaPlugin") -project(':AcaPlugin').projectDir = new File('plugins/buildsystem') +//include(":AcaPlugin") +//project(':AcaPlugin').projectDir = new File('plugins/buildsystem') include(":broker4j") project(':broker4j').projectDir = new File('broker/broker4j') -include(":LinuxBroker") -project(':LinuxBroker').projectDir = new File('broker/LinuxBroker') +//include(":LinuxBroker") +//project(':LinuxBroker').projectDir = new File('broker/LinuxBroker') include(':LabApiUtilities') project(':LabApiUtilities').projectDir = new File('common/LabApiUtilities') -include(':java-linux-test-app') -project(':java-linux-test-app').projectDir = new File('broker/java-linux-test-app') +//include(':java-linux-test-app') +//project(':java-linux-test-app').projectDir = new File('broker/java-linux-test-app') -include(":LinuxBrokerPackage") -project(':LinuxBrokerPackage').projectDir = new File('/broker/LinuxBrokerPackage') +//include(":LinuxBrokerPackage") +//project(':LinuxBrokerPackage').projectDir = new File('/broker/LinuxBrokerPackage') include(":AzureSample") project(':AzureSample').projectDir = new File('azuresample/app') @@ -219,8 +219,8 @@ project(':mockltw').projectDir = new File('broker/mockbrokers/mockltw') include(":mockbrokerapplib") project(':mockbrokerapplib').projectDir = new File('broker/mockbrokers/mockbrokerapplib') -include(":NativeAuthSample") -project(':NativeAuthSample').projectDir = new File('nativeauthsample/app') +//include(":NativeAuthSample") +//project(':NativeAuthSample').projectDir = new File('nativeauthsample/app') // Authenticator App Projects — only included when the opt-in flag is set. if (includeAuthenticatorApp) { From 19688fdb02ce0275a4562d7ab2029bb0e9c79dc6 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Wed, 5 Aug 2026 16:24:42 -0700 Subject: [PATCH 02/90] Implement inter-process state locking for CLI commands to prevent race conditions and ensure safe parallel execution; enhance error messaging for Azure DevOps authentication issues. --- release-agent/orchestrator/cli.py | 7 ++- release-agent/orchestrator/cli_common.py | 76 ++++++++++++++++++++++++ release-agent/orchestrator/engine.py | 5 +- release-agent/orchestrator/render.py | 13 ++++ release-agent/phases/agents/preflight.py | 19 +++++- release-agent/skill/SKILL.md | 8 ++- release-agent/tests/test_engine.py | 70 +++++++++++++++++++++- release-agent/tools/checks.py | 17 +++++- 8 files changed, 205 insertions(+), 10 deletions(-) diff --git a/release-agent/orchestrator/cli.py b/release-agent/orchestrator/cli.py index 49f7e464..d39e49a6 100644 --- a/release-agent/orchestrator/cli.py +++ b/release-agent/orchestrator/cli.py @@ -46,7 +46,12 @@ def build_parser(): def main(argv=None): args = build_parser().parse_args(argv) - return args.func(args) + # Serialize state read-modify-write per release so parallel CLI invocations + # (e.g. the skill firing record-step calls at once) can't clobber each other. + runs_root = getattr(args, "runs_root", None) + release = C.effective_release(runs_root, getattr(args, "release", None)) + with C.state_lock(runs_root, release): + return args.func(args) if __name__ == "__main__": diff --git a/release-agent/orchestrator/cli_common.py b/release-agent/orchestrator/cli_common.py index ade5a53b..101c829e 100644 --- a/release-agent/orchestrator/cli_common.py +++ b/release-agent/orchestrator/cli_common.py @@ -12,6 +12,8 @@ from __future__ import annotations import os +import time +from contextlib import contextmanager from orchestrator.state import ReleaseState from orchestrator.engine import Orchestrator @@ -28,6 +30,80 @@ # runs live OUTSIDE release-agent/, in android-complete/.release-runs (gitignored) DEFAULT_RUNS_ROOT = os.path.join(os.path.dirname(ROOT), ".release-runs") +# ---- inter-process state lock ---- +_LOCK_TIMEOUT = 30.0 # max seconds to wait for another CLI process to release +_LOCK_STALE = 120.0 # a lock older than this is treated as abandoned (crashed proc) + + +@contextmanager +def state_lock(runs_root: str, release): + """Serialize a release's state read-modify-write ACROSS CLI processes. + + Every mutating command loads state, mutates, then saves. Two running at once + (e.g. the skill firing `record-step` calls in parallel, or an hourly `tick` + overlapping an interactive command) would clobber each other — a last-writer- + wins lost update. This exclusive per-release lock makes each CLI invocation + atomic: a second process blocks until the first has saved and released. + Read-only commands hold it only for their brief duration. + + No release (e.g. `list`, `infra`) → no lock: nothing release-scoped to guard. + A lock older than _LOCK_STALE is stolen (its owner crashed). + """ + if not release: + yield + return + lock_dir = os.path.join(runs_root, release) + os.makedirs(lock_dir, exist_ok=True) + lock_path = os.path.join(lock_dir, ".state.lock") + deadline = time.monotonic() + _LOCK_TIMEOUT + fd = None + while True: + try: + fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.write(fd, str(os.getpid()).encode()) + break + except FileExistsError: + try: + if time.time() - os.path.getmtime(lock_path) > _LOCK_STALE: + os.remove(lock_path) # abandoned by a crashed process + continue + except OSError: + pass + if time.monotonic() > deadline: + raise TimeoutError( + f"could not acquire state lock for release {release} within " + f"{_LOCK_TIMEOUT:.0f}s — another CLI process is holding it") + time.sleep(0.05) + try: + yield + finally: + try: + os.close(fd) + except OSError: + pass + try: + os.remove(lock_path) + except OSError: + pass + + +def effective_release(runs_root, release): + """The release id to lock on. The explicit `--release` when given; otherwise, + for discovery-mode mutating commands (e.g. the hourly `tick`), the single + active release so it's still serialized against interactive commands. Returns + None when ambiguous / none exist (nothing to serialize on).""" + if release: + return release + if not runs_root: + return None + try: + res = discovery.resolve(runs_root, None) + if res.get("resolution") == "one" and res.get("release"): + return res["release"].get("release_id") + except Exception: + pass + return None + # ---- paths / state ---- def state_path(runs_root: str, release: str) -> str: diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 37207370..0cf05d76 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -501,8 +501,9 @@ def _active_phase_report(self) -> Optional[dict]: steps_view = [] for s in steps: sid = s["id"] + stp = self.state.get_step(phase["id"], sid) s_done = self.state.is_done(phase["id"], sid) - s_blocked = self.state.get_step(phase["id"], sid).status == "blocked" + s_blocked = stp.status == "blocked" is_gate = bool(s.get("gate")) is_rem = self._is_reminder(s) is_scout = s.get("source") == "scout" @@ -523,6 +524,7 @@ def _active_phase_report(self) -> Optional[dict]: steps_view.append({ "id": sid, "name": s["name"], "status": status, "needs_owner": needs, + "note": stp.note, # agent result / block reason / detail "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_scout or is_attest or s_blocked)), }) opens = self._phase_anchor_date(phase) @@ -607,6 +609,7 @@ def _current_steps(self, current_phase_obj) -> list: "reminder": self._is_reminder(s), "owner": s.get("owner", "agent"), "state": s_state, + "note": rec.get("note"), # agent result / block reason / detail }) return out diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index a8119eda..5c393b4d 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -174,6 +174,19 @@ def status_view(r: dict) -> str: tag = " 🚦" if s["gate"] else (" 📌" if s.get("reminder") else "") lines.append(f"| {icon} | {s['name']}{tag} | {word} |") + # 3b) Results & activity — surface each step's stored outcome (agent report, + # created link, block reason) so a done/blocked step isn't just "Done" with + # no evidence. Full multi-line notes (e.g. the CG report) are preserved. + results = [s for s in r["current_steps"] + if s.get("note") and s["state"] in ("done", "blocked")] + if results: + lines += ["", "### Results & activity"] + for s in results: + mark = "⛔" if s["state"] == "blocked" else "✅" + note_lines = [ln.rstrip() for ln in str(s["note"]).strip().split("\n")] + body = " \n".join(ln for ln in note_lines if ln) # markdown hard breaks + lines += ["", f"{mark} **{s['name']}** ", body] + return "\n".join(lines) diff --git a/release-agent/phases/agents/preflight.py b/release-agent/phases/agents/preflight.py index d7d41d55..2b3c1fc0 100644 --- a/release-agent/phases/agents/preflight.py +++ b/release-agent/phases/agents/preflight.py @@ -132,6 +132,12 @@ def _page_name(state, n: int = 1) -> str: return f"{base} {n} Release" if n and n >= 2 else f"{base} Release" +def _wiki_url(org: str, project: str, wiki: str, path: str) -> str: + """Browser URL for an ADO wiki page (so the result links to the created page).""" + from urllib.parse import quote + return f"{(org or '').rstrip('/')}/{project}/_wiki/wikis/{wiki}?pagePath={quote(path or '')}" + + def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: cfg = _load_cfg().get("wiki", {}) org = cfg.get("org") @@ -144,7 +150,8 @@ def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult return StepResult( True, f"[dry-run] Would create payload wiki subpage '{base_name}' under " - f"'{parent}' (duplicate-safe: a second numbered page if it already exists).", + f"'{parent}' (duplicate-safe: a second numbered page if it already exists).\n" + f"Would live at: {_wiki_url(org, project, wiki, base_path)}", "agent", ) if not (org and project and wiki and parent): @@ -165,7 +172,8 @@ def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult return StepResult( True, f"⚠ A payload page already exists for this month ('{base_name}'). " - f"Left it untouched and created a SECOND page: '{cand_name}'. ({res.detail})", + f"Left it untouched and created a SECOND page: '{cand_name}'.\n" + f"Link: {_wiki_url(org, project, wiki, cand_path)}", "agent", ) n += 1 @@ -174,7 +182,12 @@ def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult res = create_wiki_page(org, project, wiki, base_path, _payload_template(state)) if not res.ok: return StepResult(False, f"wiki: could not create '{base_path}' — {res.detail}", "agent") - return StepResult(True, f"Payload wiki subpage ready: '{base_name}' ({res.detail})", "agent") + return StepResult( + True, + f"Payload wiki subpage ready: '{base_name}'.\n" + f"Link: {_wiki_url(org, project, wiki, base_path)}", + "agent", + ) # ---- Component Governance alerts (report-only) ----------------------------- diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index 39506dfa..def9ebb4 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -127,7 +127,7 @@ If any item is unsatisfied the gate stays closed. If the engineer can't satisfy Flow after starting: 1. `python -m orchestrator.cli checklist --release --verify` — this runs the auto checks AND prints the **canonical checklist table (markdown)**. **Reproduce its stdout into your reply as live markdown (NOT wrapped in a ``` code fence)** so Scout renders it as a real table — it is already a finished markdown table with the type labels, per-item status, and clickable links. **Do NOT rebuild, re-format, re-order, re-label, or re-type any of it from memory, and do NOT fence it.** If you reconstruct it you WILL introduce errors (stale icons, mangled/merged URLs); if you fence it, it shows as raw text. Always reproduce the literal command output as rendered markdown. You may add a sentence of your own before or after, but the table block itself must match the output. 2. There are exactly **two types by resolver**: `[auto]` (Scout verifies) and `[attest]` (the user confirms). All items must be satisfied to clear the gate. (Do not add lock icons or a "hard requirement" legend.) -3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. +3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. **Do this quietly: run the checks and `record-check` them without narrating each one** (a wall of per-check prose is what buries the table). The only time 3a surfaces anything to the user is the `silent_perms` opt-out choice below. - **`oncall_now` (ICM):** call the ICM MCP `get_on_call_schedule_by_team_id` with `teamIds: [78848]` ("Auth Client Android Shield"). Resolve the current user's alias (`get_my_icm_context` or the owner email's local part), then decide by their role in `shiftCurrentOnCalls[].currentOnCallContacts[]`: - **Not in the roster at all** → `record-check --item oncall_now --status pass --detail "not on the current roster"`. - **Present but NOT the primary** (i.e. they are a **backup/secondary** — any position other than the first-listed contact) → **pass**: `record-check --item oncall_now --status pass --detail "backup OCE, not primary (primary: )"`. A backup is free to run the release. @@ -142,7 +142,7 @@ Flow after starting: - They pick **Enable** → the only manual step is the Scout master toggle: if `permissions.allowModelPermissionsChange` is `false`, tell them to turn on **Settings → Permissions → "Allow AI to request permission changes"** (I cannot flip it — it's read-only from the model, by design). Once it's `true`, call **`m_request_permission_escalation`** with `servers: { workiq: {autoApprove:true}, playwright: {autoApprove:true} }` (add `shell` if off too); they click **Allow** once, then re-read `m_get_settings` and `record-check … --status pass --detail "enabled silent runs"`. - They pick **Proceed without** (or won't enable the master toggle) → `record-check --release --item silent_perms --status degraded --detail "proceeding without silent runs — unattended digest/Teams/browser checks will prompt & may stall until Scout is opened"`. **`degraded` satisfies the gate** (the checklist shows it as ⚠️ *Proceeding (not silent)*), so the release can start; the downside is on record. - A `fail` on `oncall_now`/`adx_access`, or a real problem, keeps the gate closed — treat it like any unsatisfiable required item (resolve or hand off). Do NOT attest these — they're `auto` items you verified. (`silent_perms` is the one soft/opt-out auto item: it uses `degraded`, never `fail`, when the user chooses to proceed.) -3b. Use `m_ask_user` to collect the **attestations**: `play_console_access`, the on-call **window** (`oncall_window` — show the CCD‑7 → CCD+14 dates from the checklist), `saw_ame`, `yubikey`. Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. +3b. **Re-anchor on the TABLE, then attest.** After recording the auto results, **run `python -m orchestrator.cli checklist --release ` again and reproduce its updated markdown table** — the auto items now show ✅ and only the attests are Outstanding. This table is the single source of truth and **must be shown immediately before you ask for attestations**; the auto-check work in 3a pushes the first table far up, so re-render it here every time. Then use `m_ask_user` to collect the four attestations (`play_console_access`; the on-call **window** `oncall_window` — the CCD‑7 → CCD+14 dates are in the table; `saw_ame`; `yubikey`). Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. **Never replace the table with a plain prose list of the items** — the `m_ask_user` prompt accompanies the re-rendered table, it does not substitute for it. 4. If they confirm everything → `sign --release --all`, then `next` to begin Phase 0. 5. **If they can't satisfy any attest item** (on-call during the window, no SAW, no YubiKey, no portal access) → `decline --release --item ` (repeat `--item` for each). The gate is now blocked. Tell them plainly: the release can't start until that item is resolved; if they can't resolve it, hand the release to another engineer who can (notify their manager / the release team). Treat every item this way — don't single any out as harder or softer. 6. If an **auto** item shows FAIL (no build-definition access, or you're on-call), the gate stays closed — a real problem to resolve, not something to attest around. @@ -154,10 +154,14 @@ Never hand-edit or regenerate the checklist/status blocks — always show the CL Some phases run **in parallel** (Phase 0 is `execution: parallel`): a single `next` runs **every independent automated step at once** (breaking, CG, cron, wiki — all complete in one call) and then surfaces **all the human/scout holds together** (e.g. *"4 item(s) need you: …"*). So don't treat it as one-step-at-a-time. After `next`, read `status --json` and look at **`pending_human`** (and `active_phase.steps` with their `status`/`needs_owner`) — that's the full set of what's outstanding. Work through **all** of them in this pass: - **`source: scout`** steps (notice, flight_reminder, lockdown) → run each via MCP/browser + `record-step` (see the sections below). These are independent — do them all. + +> **State writes are safe to parallelize.** The CLI serializes every state read-modify-write per release with an exclusive lock, so firing several `record-step` / `record-check` / `done` calls at once (or an hourly `tick` overlapping your command) can't clobber each other — a second invocation simply waits for the first to save. You don't need to run them one-at-a-time to avoid races. - **`attest`** steps (confirm_reminders, vitals) → ask the owner to confirm, then `done --step `. - **`blocked`** steps (cg/cron on a real problem) → show the note; fix + rerun, or skip. Dependencies still hold: `confirm_reminders` only appears **after** `flight_reminder` is sent (it won't be in `pending_human` until then). Call `next` again after clearing holds to let newly-ready steps surface and, once all are done, advance to the next phase. +> **ALWAYS show the `status` table each iteration — never a bare prose list.** After every `next`, and after every `done`/`record-step` that clears a hold, run `python -m orchestrator.cli status --release ` (no `--json`, add `--as-of` if simulating) and **reproduce its full output as live markdown**: the phase map, the current-phase **steps table** (what's done ✅ / pending / needs you 📌), and the **Results & activity** section (each agent's result — the CG report, the created **wiki link**, block reasons). This is the source of truth the user asked for; the human-readable `status` also auto-logs what was shown. Use `status --json` only for your own branching — but still show the rendered table to the user. Then, alongside the table, tell the user exactly what each outstanding step needs (run the scout steps yourself; for the `attest` steps spell out what to confirm). Do **not** replace the table with your own summarized list of remaining steps. + ## Scout-assisted phase steps (CCOA lockdown check) Some Phase steps read AAD-gated sources the deterministic engine can't reach, so **you run them via the browser and record the result** — same idea as the readiness scout checks, but mid-phase. When advancing, if **`lockdown`** is among the pending holds (`source: scout`, in `pending_human`), handle it like this — silently, without bothering the user unless there's an overlap: diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 0b42df0f..f95db780 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -853,8 +853,76 @@ def test_registry_register_list_deregister(): assert len(reg.list()) == 1 -# ---- Phase-0 real pre-flight agents (breaking detect, wiki payload) ---- +# ---- inter-process state lock (parallel CLI mutation safety) ---- + +def test_state_lock_is_exclusive_then_releases(): + """While a release's state lock is held, a second acquisition blocks (times + out); once released it can be acquired again.""" + import threading + with tempfile.TemporaryDirectory() as rr: + R = "2099-02" + os.makedirs(os.path.join(rr, R)) + acquired, timed_out = [], [] + with C.state_lock(rr, R): + orig = C._LOCK_TIMEOUT + C._LOCK_TIMEOUT = 0.3 + def try_acquire(): + try: + with C.state_lock(rr, R): + acquired.append(True) + except TimeoutError: + timed_out.append(True) + t = threading.Thread(target=try_acquire) + t.start(); t.join() + C._LOCK_TIMEOUT = orig + assert timed_out and not acquired # blocked while held + with C.state_lock(rr, R): # released -> acquirable + acquired.append("after") + assert "after" in acquired + + +def test_concurrent_record_check_both_persist(): + """Two record-check CLI invocations fired at the same instant must BOTH + persist — the per-release lock prevents the last-writer-wins clobber that + dropped `notice` in the live test (parallel state-write race).""" + import threading + from orchestrator import cli as _cli + with tempfile.TemporaryDirectory() as rr: + R = "2099-03" + _cli.main(["--runs-root", rr, "init", "--release", R, + "--owner-email", "t@example.com", "--owner-name", "T"]) + barrier = threading.Barrier(2) + + def rec(item): + barrier.wait() # maximize overlap + _cli.main(["--runs-root", rr, "record-check", "--release", R, + "--item", item, "--status", "pass", "--detail", item]) + threads = [threading.Thread(target=rec, args=(i,)) + for i in ("oncall_now", "adx_access")] + for t in threads: + t.start() + for t in threads: + t.join() + st = C.load_state(rr, R) + assert st.readiness_items.get("oncall_now", {}).get("status") == "pass" + assert st.readiness_items.get("adx_access", {}).get("status") == "pass" + + +def test_status_surfaces_agent_result_notes_and_wiki_link(): + """Agent results are stored as the step note and surfaced in status: the + Results & activity section shows each done agent's output, incl. the wiki link.""" + from orchestrator import render + st, orch = _ccd_orch("2026-07-02") # Phase 0 open + orch.run_until_gate() # runs breaking/cg/cron/wiki (set notes) + r = orch.status_report() + steps = {s["id"]: s for s in r["current_steps"]} + assert steps["cg"].get("note") and steps["wiki"].get("note") + view = render.status_view(r) + assert "Results & activity" in view + assert "Would live at" in view # the wiki link surfaces (dry-run) + +# ---- Phase-0 real pre-flight agents (breaking detect, wiki payload) ---- _SAMPLE_CHANGELOG = """vNext ---------- - [MINOR] add a thing (#1) diff --git a/release-agent/tools/checks.py b/release-agent/tools/checks.py index 677f09e7..e490d504 100644 --- a/release-agent/tools/checks.py +++ b/release-agent/tools/checks.py @@ -38,9 +38,22 @@ def check_ado_build_def(org: str, project: str, def_id: int, timeout: int = 30) return CheckResult(False, False, f"failed to run az: {e}") if out.returncode == 0 and out.stdout.strip(): return CheckResult(True, True, f"accessible: '{out.stdout.strip()}'") - err = (out.stderr or "").strip().splitlines() + err_text = (out.stderr or "").strip() + low = err_text.lower() + # Distinguish a real ACCESS problem from an az-not-authenticated-to-ADO state + # (org rejects the CLI's AAD token — often Conditional Access — and returns the + # sign-in page / 401). Give an actionable message instead of a raw 401. + if ("requires user authentication" in low or "unauthorized" in low + or "sign in" in low or "tf400813" in low or "no credentials" in low): + return CheckResult( + False, True, + f"az is not authenticated to Azure DevOps (build def {def_id}); the org " + f"rejected the CLI token. Run `az login`; if it still fails (Conditional " + f"Access), sign in with a PAT: `az devops login --organization {org}` " + f"(Build: Read scope).") + err = err_text.splitlines() msg = err[-1] if err else f"cannot access build def {def_id}" - return CheckResult(False, True, msg[:160]) + return CheckResult(False, True, msg[:200]) def current_az_user(timeout: int = 20): From d5ea1d4a140a3deb11342da81352d3286333add6 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Wed, 5 Aug 2026 16:26:36 -0700 Subject: [PATCH 03/90] Ignore local pipeline and wiki repositories --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2802f33f..733b31d1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ tsl msalcpp design-docs 1ES-Pipelines +AuthClientAndroidPipelines +IdentityWiki.wiki nativeauthsample # Gradle files From dc5d8ab27f3c3ca072cba12aa50ae695b075bd63 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 13 Aug 2026 20:47:57 +0100 Subject: [PATCH 04/90] release-agent: harden readiness gate + modularize the skill Engine (gate integrity): - sign: remove blanket --all; CLI now requires explicit --item ids + refuses a bare sign, and records each attestation individually with an evidence --note. Closes the hole where sign --all attested every human item in one blind call (a release reached Phase 0 without real confirmation). - readiness.sign() carries a per-item evidence note. Readiness config/UX: - silent_perms.required_servers now includes kusto + icm (the MCP servers the adx_access / oncall_now checks call) so they don't prompt on first run. Skill (A: dedupe, B: modular split): - Split the 48KB monolith SKILL.md into a lean 7KB core (golden rules + behaviour dispatch + reference routing table) plus skill/reference/*.md read on demand (readiness-gate, starting-and-scheduling, presenting-status, commands, phases/preflight, phases/_TEMPLATE). Core is now well under Scout's inline limit, so it no longer spills / gets skimmed. - Fix regression where the readiness table was suppressed: render the checklist table FIRST, then handle silent-runs (was "before showing the checklist"). - Anti-assumption rule: never attest/approve on an echoed m_ask_user result. - Scale convention: add a phase = phases.yaml + phases/agents/.py + reference/phases/.md + one routing-table row. bootstrap.ps1: auto-install Python via winget + silent pyyaml; verify Scout present + skill copy; restart Scout by default (guarded when run inside a Scout session); UTF-8 + detached-launch fixes; clearer folder/run instructions. Tests: 101/101 (added regressions for bare-sign refusal, no --all, evidence note). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/README.md | 8 +- release-agent/config/readiness.yaml | 8 +- .../orchestrator/commands/readiness.py | 36 +- release-agent/orchestrator/readiness.py | 13 +- release-agent/setup/bootstrap.ps1 | 235 +++++++++++-- release-agent/skill/SKILL.md | 324 +++--------------- release-agent/skill/reference/commands.md | 52 +++ .../skill/reference/phases/_TEMPLATE.md | 28 ++ .../skill/reference/phases/preflight.md | 54 +++ .../skill/reference/presenting-status.md | 14 + .../skill/reference/readiness-gate.md | 60 ++++ .../reference/starting-and-scheduling.md | 70 ++++ release-agent/tests/test_engine.py | 41 ++- 13 files changed, 625 insertions(+), 318 deletions(-) create mode 100644 release-agent/skill/reference/commands.md create mode 100644 release-agent/skill/reference/phases/_TEMPLATE.md create mode 100644 release-agent/skill/reference/phases/preflight.md create mode 100644 release-agent/skill/reference/presenting-status.md create mode 100644 release-agent/skill/reference/readiness-gate.md create mode 100644 release-agent/skill/reference/starting-and-scheduling.md diff --git a/release-agent/README.md b/release-agent/README.md index be2932ad..7dc8fdfb 100644 --- a/release-agent/README.md +++ b/release-agent/README.md @@ -93,9 +93,13 @@ The conductor is **stateless**: on each invocation it loads the record, (later) ## Quick start +Run the one-time setup from the **`release-agent` folder of your `android-complete` clone**, +using PowerShell 7 (`pwsh`): + ```powershell -# one-time -pwsh ./setup/bootstrap.ps1 +# one-time — from the release-agent folder +cd C:\repos\android-complete\release-agent # adjust to your clone location +pwsh .\setup\bootstrap.ps1 ``` `bootstrap.ps1` runs an **infrastructure preflight** first (`python -m orchestrator.cli infra`), diff --git a/release-agent/config/readiness.yaml b/release-agent/config/readiness.yaml index bd9edd3c..add4d6a1 100644 --- a/release-agent/config/readiness.yaml +++ b/release-agent/config/readiness.yaml @@ -64,13 +64,15 @@ items: - id: silent_perms label: "Silent-run permissions" - text: "Permissions allow fully unattended runs — shell, WorkIQ (email + Teams), and the browser are auto-approved so scheduled work never stalls on a prompt" - detail: "Scout checks shell, WorkIQ (email + Teams), and the browser are all auto-approved so the daily automation runs silently when Scout isn't focused." + text: "Permissions allow fully unattended runs — shell, WorkIQ (email + Teams), the browser, and the ICM + Kusto/ADX MCP servers are auto-approved so scheduled work and the on-call/telemetry checks never stall on a prompt" + detail: "Scout checks shell, WorkIQ (email + Teams), the browser, and the ICM + Kusto/ADX MCP servers are all auto-approved so the daily automation and the readiness on-call/telemetry checks run silently when Scout isn't focused." verify: auto source: scout # only the skill can read Scout's own settings (m_get_settings) verifier: silent_perms # The servers that must be auto-approved for a fully-silent run (kept as data). - required_servers: [shell, workiq, playwright] + # Includes the MCP servers the readiness auto-checks call: `kusto` (adx_access) and + # `icm` (oncall_now) — otherwise those checks prompt on first run. + required_servers: [shell, workiq, playwright, kusto, icm] # OPT-OUT (soft): enabling silent runs needs the user to turn on the Scout master # toggle "Allow AI to request permission changes" (allowModelPermissionsChange) — # which ONLY the user can flip in the UI. The skill offers to enable silent runs; diff --git a/release-agent/orchestrator/commands/readiness.py b/release-agent/orchestrator/commands/readiness.py index e4a75137..5b6786c6 100644 --- a/release-agent/orchestrator/commands/readiness.py +++ b/release-agent/orchestrator/commands/readiness.py @@ -36,17 +36,35 @@ def cmd_verify(args): def cmd_sign(args): st, orch = C.load_orch(args.runs_root, args.release, args.config) - ids = None if args.all else (args.item or []) - chk = orch.gate.sign(ids) + ids = args.item or [] + if not ids: + # No blanket sign: the engineer must name the item(s) they confirmed. + # This closes the integrity hole where `sign --all` attested every human + # item in one blind call with no evidence and no per-item confirmation. + print("Refusing to sign: name the item(s) you confirmed with --item " + "(repeatable). Attest only what the engineer explicitly confirmed.") + return 2 + # Validate the ids are real attest items before recording anything. + chk_before = orch.gate.checklist() + attest_ids = {i["id"] for i in chk_before["attest_items"]} + unknown = [i for i in ids if i not in attest_ids] + if unknown: + print(f"Not attestable (unknown or not an attest item): {', '.join(unknown)}") + return 2 + chk = orch.gate.sign(ids, note=args.note or None) C.save_state(st, args.runs_root, args.release) - C.elog(args.runs_root, args.release).log( - "readiness_signed" if chk["signed"] else "readiness_partial", - items=("all" if ids is None else ids), signed=chk["signed"]) + el = C.elog(args.runs_root, args.release) + # Log EACH attestation individually with its evidence note, so the trail shows + # exactly what was confirmed (not one opaque items:"all"). + for iid in ids: + el.log("readiness_attested", item=iid, driver=args.note or None) + el.log("readiness_signed" if chk["signed"] else "readiness_partial", + items=ids, signed=chk["signed"]) if chk["signed"]: print(f"Readiness signed at {chk['signed_at']}. Entry gate cleared — you can now start Phase 0.") else: pending = [i["id"] for i in chk["items"] if not i["satisfied"]] - print(f"Recorded. Still pending: {', '.join(pending)}") + print(f"Recorded {', '.join(ids)}. Still pending: {', '.join(pending)}") return 0 @@ -110,8 +128,10 @@ def register(sub): sg = sub.add_parser("sign", help="Attest human readiness items (also runs auto verify)") sg.add_argument("--release", required=True) - sg.add_argument("--all", action="store_true", help="Attest every human item") - sg.add_argument("--item", action="append", help="Attest a specific item id (repeatable)") + sg.add_argument("--item", action="append", + help="Attest a specific item id the engineer confirmed (repeatable, required)") + sg.add_argument("--note", default="", + help="Evidence: what the engineer confirmed (recorded per item)") sg.set_defaults(func=cmd_sign) dc = sub.add_parser("decline", help="Declare you CANNOT satisfy an item (may block ownership)") diff --git a/release-agent/orchestrator/readiness.py b/release-agent/orchestrator/readiness.py index 2d35ccde..87838789 100644 --- a/release-agent/orchestrator/readiness.py +++ b/release-agent/orchestrator/readiness.py @@ -145,10 +145,12 @@ def record_check(self, item_id: str, status: str, message: str = "") -> dict: self._refresh_signed() return self.checklist() - def sign(self, item_ids=None) -> dict: + def sign(self, item_ids=None, note=None) -> dict: """Attest human (attest) items and run auto verifiers. item_ids=None - attests every attest item. Auto items are only set by verification — - they cannot be hand-waved through. Signs when all items are satisfied.""" + attests every attest item (library convenience — the CLI never does this; + it requires explicit item ids). Auto items are only set by verification — + they cannot be hand-waved through. `note` records the human's confirmation + as evidence on each attested item. Signs when all items are satisfied.""" if not self.config: self.state.readiness_signed = True self.state.readiness_signed_at = _now() @@ -158,7 +160,10 @@ def sign(self, item_ids=None) -> dict: attest_ids = [it["id"] for it in all_items if it.get("verify", "attest") == "attest"] targets = attest_ids if item_ids is None else [i for i in item_ids if i in attest_ids] for iid in targets: - self.state.readiness_items[iid] = {"status": "attested", "at": _now()} + rec = {"status": "attested", "at": _now()} + if note: + rec["note"] = note + self.state.readiness_items[iid] = rec self._refresh_signed() return self.checklist() diff --git a/release-agent/setup/bootstrap.ps1 b/release-agent/setup/bootstrap.ps1 index 44a53189..f56433a2 100644 --- a/release-agent/setup/bootstrap.ps1 +++ b/release-agent/setup/bootstrap.ps1 @@ -8,24 +8,107 @@ Steps: 1. Infrastructure preflight — check CLIs/host deps AND register + verify the MCP servers the skill needs inside Scout (from config/requirements.yaml). - 2. Install the /release-agent skill into the Scout skills folder. - 3. Print next steps. + If Python 3.9+ is missing it is installed automatically via winget + (per-user, no admin); PyYAML is then installed via pip. Pass + -NoAutoInstallPython to opt out of the Python auto-install. + 2. Install the /release-agent skill into the Scout skills folder — only if + Microsoft Scout is detected (~/.scout present); the copy is then verified. + 3. Print next steps. By default the script closes and relaunches Scout so the + new skill / MCP servers load (skipped automatically when this setup is run + from inside a Scout session, to avoid killing itself). Pass -NoRestartScout + to opt out and restart Scout yourself. + + HOW TO RUN + Run this from the `release-agent` folder of your android-complete clone, + using PowerShell 7 (pwsh): + + cd \android-complete\release-agent + pwsh .\setup\bootstrap.ps1 + + Example (default clone location): + + cd C:\repos\android-complete\release-agent + pwsh .\setup\bootstrap.ps1 + + You can launch it from any working directory as long as you give the full + path to the script (the script locates its own folder), e.g.: + + pwsh C:\repos\android-complete\release-agent\setup\bootstrap.ps1 .EXAMPLE - pwsh ./setup/bootstrap.ps1 + cd C:\repos\android-complete\release-agent + pwsh .\setup\bootstrap.ps1 #> [CmdletBinding()] param( [string]$ScoutSkillsDir = "$env:USERPROFILE\.scout\m-skills", - [switch]$SkipSkillInstall + [switch]$SkipSkillInstall, + [switch]$NoAutoInstallPython, # by default, if Python is missing we install it via winget + [switch]$NoRestartScout # by default we restart Scout so new skill/MCP servers load ) $ErrorActionPreference = "Stop" + +# The Python engine prints UTF-8 (em-dashes, etc.). Make sure this console reads/writes +# UTF-8 so echoed output doesn't turn into mojibake like "ΓÇö". +try { + [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 + $OutputEncoding = [System.Text.Encoding]::UTF8 + $env:PYTHONIOENCODING = "utf-8" + $env:PYTHONUTF8 = "1" +} catch {} + $AgentRoot = Split-Path -Parent $PSScriptRoot # release-agent/ $RepoRoot = Split-Path -Parent $AgentRoot # android-complete/ $ReqFile = Join-Path $AgentRoot "config\requirements.yaml" Write-Host "Release Orchestrator bootstrap`n" -ForegroundColor Cyan +Write-Host " Running from: $AgentRoot" -ForegroundColor DarkGray +Write-Host " (run this from the 'release-agent' folder of your android-complete clone)`n" -ForegroundColor DarkGray + +# Sanity check: make sure we're actually in the release-agent folder. +if (-not (Test-Path (Join-Path $AgentRoot 'setup\bootstrap.ps1'))) { + Write-Host " This does not look like the release-agent folder." -ForegroundColor Red + Write-Host " cd into \android-complete\release-agent and run: pwsh .\setup\bootstrap.ps1`n" -ForegroundColor Red + exit 1 +} + +# True when THIS script was launched from a shell inside a Scout session — Scout sets +# these env vars for its child processes. Restarting Scout from there would kill us. +$InsideScout = [bool]($env:COPILOT_AGENT_SESSION_ID -or $env:COPILOT_CLI) + +# Stop all Scout processes and relaunch the app. Returns $true if it relaunched. +function Restart-Scout { + $procs = @(Get-Process -Name 'scout' -ErrorAction SilentlyContinue) + $exe = ($procs | Where-Object { $_.Path } | Select-Object -First 1).Path + if (-not $exe) { $exe = Join-Path $env:LOCALAPPDATA 'Programs\Microsoft Scout\scout.exe' } + if (-not (Test-Path $exe)) { + Write-Host " Could not locate scout.exe to relaunch." -ForegroundColor Yellow + return $false + } + if ($procs.Count) { + Write-Host " Closing Scout ($($procs.Count) process(es)) ..." -ForegroundColor DarkYellow + $procs | ForEach-Object { try { Stop-Process -Id $_.Id -Force -ErrorAction Stop } catch {} } + # Wait for the processes to actually exit before relaunching (single-instance lock). + for ($i = 0; $i -lt 20 -and (Get-Process -Name 'scout' -ErrorAction SilentlyContinue); $i++) { + Start-Sleep -Milliseconds 250 + } + } + Write-Host " Relaunching Scout ..." -ForegroundColor DarkYellow + # Launch fully DETACHED so Scout does NOT inherit this console — otherwise its + # Electron startup logs spill into the terminal after setup returns. Win32_Process + # Create starts it in a brand-new process with no console attachment. + $launched = $false + try { + $r = Invoke-CimMethod -ClassName Win32_Process -MethodName Create -Arguments @{ CommandLine = "`"$exe`"" } -ErrorAction Stop + if ($r.ReturnValue -eq 0) { $launched = $true } + } catch {} + if (-not $launched) { + # Fallback: explorer.exe launches the app as its child, also detached from our console. + try { Start-Process explorer.exe -ArgumentList "`"$exe`""; $launched = $true } catch {} + } + return $launched +} # ---- 1. Infrastructure preflight (CLIs + MCP servers), data-driven ---- # Delegates to the engine (python -m orchestrator.cli infra), which reads @@ -35,52 +118,156 @@ Write-Host "Release Orchestrator bootstrap`n" -ForegroundColor Cyan Write-Host "1. Infrastructure preflight (from config/requirements.yaml)" if (-not (Test-Path $ReqFile)) { Write-Host " requirements.yaml not found at $ReqFile" -ForegroundColor Red; exit 1 } +# Resolve a working Python (3.9+). Prefer `python`/`python3`, fall back to `py -3`, +# then probe well-known winget/python.org install dirs (PATH may be stale in this session). +function Resolve-Python { + foreach ($cand in @('python','python3')) { + try { & $cand -c "import sys; sys.exit(0 if sys.version_info[:2] >= (3,9) else 1)" 2>$null; if ($LASTEXITCODE -eq 0) { return $cand } } catch {} + } + try { & py -3 -c "import sys; sys.exit(0 if sys.version_info[:2] >= (3,9) else 1)" 2>$null; if ($LASTEXITCODE -eq 0) { return 'py -3' } } catch {} + # Direct probe of standard per-user install locations (newest first) + $globs = @( + "$env:LOCALAPPDATA\Programs\Python\Python3*\python.exe", + "$env:ProgramFiles\Python3*\python.exe" + ) + foreach ($g in $globs) { + $hit = Get-ChildItem $g -ErrorAction SilentlyContinue | Sort-Object FullName -Descending | Select-Object -First 1 + if ($hit) { + try { & $hit.FullName -c "import sys; sys.exit(0 if sys.version_info[:2] >= (3,9) else 1)" 2>$null; if ($LASTEXITCODE -eq 0) { return "`"$($hit.FullName)`"" } } catch {} + } + } + return $null +} + +$PyExe = Resolve-Python + +# If Python is missing, install it for the user via winget (per-user, no admin), then re-resolve. +if (-not $PyExe -and -not $NoAutoInstallPython) { + $winget = (Get-Command winget -ErrorAction SilentlyContinue) + if ($winget) { + Write-Host " [ ] Python 3.9+ not found - installing Python 3.12 via winget (no admin needed) ..." -ForegroundColor DarkYellow + winget install --id Python.Python.3.12 -e --source winget --scope user ` + --accept-package-agreements --accept-source-agreements --disable-interactivity 2>&1 | + ForEach-Object { Write-Host " $_" -ForegroundColor DarkGray } + # Refresh PATH from the registry so the just-installed python is visible this session. + $env:Path = [Environment]::GetEnvironmentVariable('Path','Machine') + ';' + + [Environment]::GetEnvironmentVariable('Path','User') + $PyExe = Resolve-Python + if ($PyExe) { Write-Host " [x] Python installed" -ForegroundColor Green } + else { Write-Host " [ ] Python installed but not detected yet - close and reopen your terminal, then re-run setup." -ForegroundColor Yellow } + } else { + Write-Host " [ ] Python 3.9+ missing and winget is unavailable for auto-install." -ForegroundColor Yellow + } +} + $haveInfra = $false -try { python -c "import yaml" 2>$null; if ($LASTEXITCODE -eq 0) { $haveInfra = $true } } catch { $haveInfra = $false } +if ($PyExe) { + # Python is present. Ensure PyYAML — install it silently via pip, no user action needed. + & cmd /c "$PyExe -c ""import yaml"" 2>nul" + if ($LASTEXITCODE -eq 0) { + $haveInfra = $true + } else { + Write-Host " [ ] PyYAML missing - installing via pip ..." -ForegroundColor DarkYellow + & cmd /c "$PyExe -m pip install --quiet --disable-pip-version-check pyyaml" + & cmd /c "$PyExe -c ""import yaml"" 2>nul" + if ($LASTEXITCODE -eq 0) { $haveInfra = $true; Write-Host " [x] PyYAML installed" -ForegroundColor Green } + } +} $ok = $true $restartNeeded = $false if (-not $haveInfra) { - Write-Host " [ ] Python + PyYAML ... MISSING (needed to run the preflight)" -ForegroundColor Yellow - Write-Host " install: Python 3.9+ then python -m pip install pyyaml" -ForegroundColor DarkYellow + if (-not $PyExe) { + Write-Host " [ ] Python 3.9+ ... still MISSING (needed to run the preflight)" -ForegroundColor Yellow + if ($NoAutoInstallPython) { + Write-Host " Auto-install was disabled (-NoAutoInstallPython)." -ForegroundColor DarkYellow + } + Write-Host " Install Python 3.9+ ('winget install Python.Python.3.12' or https://aka.ms/python)," -ForegroundColor DarkYellow + Write-Host " reopen your terminal, then re-run setup." -ForegroundColor DarkYellow + } else { + Write-Host " [ ] PyYAML ... could not be installed automatically" -ForegroundColor Yellow + } $ok = $false } else { Push-Location $AgentRoot try { - $out = python -m orchestrator.cli infra 2>&1 + $out = & cmd /c "$PyExe -m orchestrator.cli infra 2>&1" $out | ForEach-Object { Write-Host " $_" } if ($LASTEXITCODE -ne 0) { $ok = $false } if ($out -match "RESTART Scout") { $restartNeeded = $true } } finally { Pop-Location } # engine config presence (cheap local sanity check) - Write-Host -NoNewline " [ ] engine config (phases.yaml) ... " - if (Test-Path (Join-Path $AgentRoot 'config\phases.yaml')) { Write-Host "OK" -ForegroundColor Green } - else { Write-Host "MISSING" -ForegroundColor Yellow; $ok = $false } + if (Test-Path (Join-Path $AgentRoot 'config\phases.yaml')) { + Write-Host " [OK] engine config (phases.yaml)" -ForegroundColor Green + } else { + Write-Host " [MISSING] engine config (phases.yaml)" -ForegroundColor Yellow; $ok = $false + } } if (-not $ok) { Write-Host "`nSome infrastructure is missing — resolve the items above, then re-run." -ForegroundColor Yellow } -if ($restartNeeded) { - Write-Host "`n>>> RESTART Scout now so the newly-registered MCP server(s) load. <<<" -ForegroundColor Cyan -} Write-Host "`n2. Skill install" if ($SkipSkillInstall) { Write-Host " Skipped (--SkipSkillInstall)." } else { - $src = Join-Path $AgentRoot "skill\SKILL.md" - $destDir = Join-Path $ScoutSkillsDir "release-agent" - New-Item -ItemType Directory -Force -Path $destDir | Out-Null - Copy-Item $src (Join-Path $destDir "SKILL.md") -Force - Write-Host " Installed /release-agent skill -> $destDir" -ForegroundColor Green + $src = Join-Path $AgentRoot "skill\SKILL.md" + $ScoutRoot = Split-Path -Parent $ScoutSkillsDir # ~/.scout + $destDir = Join-Path $ScoutSkillsDir "release-agent" + $destFile = Join-Path $destDir "SKILL.md" + + if (-not (Test-Path $src)) { + Write-Host " [ ] Source skill not found at $src" -ForegroundColor Red + $ok = $false + } + # Is Scout actually installed? Its per-user data folder (~/.scout) is created on + # first launch. If it's absent, Scout isn't installed/run yet — don't fabricate it. + elseif (-not (Test-Path $ScoutRoot)) { + Write-Host " [ ] Microsoft Scout not detected ($ScoutRoot is missing)." -ForegroundColor Yellow + Write-Host " Install Microsoft Scout and launch it once, then re-run this setup" -ForegroundColor DarkYellow + Write-Host " to install the /release-agent skill." -ForegroundColor DarkYellow + $ok = $false + } + else { + New-Item -ItemType Directory -Force -Path $destDir | Out-Null + Copy-Item $src $destFile -Force + # Verify the skill actually landed (exists + non-empty + size matches source). + $srcLen = (Get-Item $src).Length + if ((Test-Path $destFile) -and ((Get-Item $destFile).Length -eq $srcLen) -and ($srcLen -gt 0)) { + Write-Host " [x] Installed /release-agent skill -> $destFile ($srcLen bytes)" -ForegroundColor Green + $skillInstalled = $true + } else { + Write-Host " [ ] Skill copy could not be verified at $destFile" -ForegroundColor Red + $ok = $false + } + } + + # Report every release-agent skill Scout can currently see. + if (Test-Path $ScoutSkillsDir) { + $installed = Get-ChildItem $ScoutSkillsDir -Directory -ErrorAction SilentlyContinue | + Where-Object { Test-Path (Join-Path $_.FullName 'SKILL.md') } | + Select-Object -ExpandProperty Name + if ($installed) { Write-Host " Skills currently installed in Scout: $($installed -join ', ')" -ForegroundColor DarkGray } + } +} + +$needsReload = ($restartNeeded -or $skillInstalled) +if ($needsReload) { + Write-Host "`nScout must reload to pick up the newly-installed skill / MCP server(s)." + if ($NoRestartScout) { + Write-Host " Auto-restart disabled (-NoRestartScout). Restart Scout yourself so the changes load." -ForegroundColor DarkYellow + } elseif ($InsideScout) { + Write-Host " Skipping auto-restart: this setup is running inside a Scout session," -ForegroundColor Yellow + Write-Host " so restarting would kill it. Close and reopen Scout manually." -ForegroundColor Yellow + } else { + $done = Restart-Scout + if ($done) { Write-Host " Scout restarted — the skill/MCP servers will load on launch." -ForegroundColor Green } + else { Write-Host " Please restart Scout manually so the changes load." -ForegroundColor Yellow } + } } Write-Host "`n3. Next steps" -ForegroundColor Cyan Write-Host " * Open Scout and run: /release-agent" -Write-Host " * Or drive the engine directly from $AgentRoot :" -Write-Host " python -m orchestrator.cli init --release 2026-07" -Write-Host " python -m orchestrator.cli next --release 2026-07" -Write-Host " python -m orchestrator.cli status --release 2026-07" -Write-Host "`n Everything runs in DRY-RUN by default. Nothing touches production.`n" +Write-Host " * The agent drives the whole release from inside Scout - just follow its prompts.`n" diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index def9ebb4..3aa13e23 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -5,282 +5,54 @@ description: Drive an Android release end-to-end using the Release Orchestrator # /release-agent — Release Orchestrator conductor -You are the conversation layer over the **Release Orchestrator engine** (deterministic Python). -The engine decides what happens next; you discover releases, present status/gates nicely, and relay decisions. -**Never decide the release flow yourself, and never invent a release** — always call the engine. +> **Recommended model:** run on a high-reasoning model (e.g. **claude-opus-4.8**). Release work involves gate decisions, Component Governance / incident judgment, and multi-step state reconciliation. Scout skills can't self-select a model, so switch the session model before invoking if you're on a lighter one. (The unattended "Release push reminders" automation is already pinned to a strong model.) -## Where things live -- Engine + config: `C:\repos\android-complete\release-agent\` (run commands from here). -- Run-state: `C:\repos\android-complete\.release-runs\\release-state.json` (gitignored; one per month, e.g. `2026-07`). -- The `setup/bootstrap.ps1` script ONLY prepares the machine. Its first step is an **infrastructure preflight** (`python -m orchestrator.cli infra`): it checks the CLIs/host deps in `config/requirements.yaml` AND registers the **MCP servers** the skill needs into Scout's config — the **ICM** server (on-call lookups) and the **Kusto/ADX** server (telemetry queries), both provided by the Agency CLI — backing the config up and telling the engineer to restart Scout. It also checks **Scout itself is installed** (`~/.scout`); if not, it stops and says to install Scout first. It does **not** start a release — that happens here, in Scout. -- **Kusto is multi-cluster:** clusters live as data under `kusto_clusters` in `config/requirements.yaml`; infra wires them all into the one Kusto MCP via `--known-services`. To make a new cluster queryable, add an entry there and re-run `python -m orchestrator.cli infra` (then restart Scout). -- If an infra check ever fails (a needed MCP server isn't registered, or Scout wasn't restarted after registering), run `python -m orchestrator.cli infra` and tell the user to restart Scout; the manifest is `config/requirements.yaml`. - -## ALWAYS discover first (the none / one / many rule) -On ANY request about a release (status, continue, approve, advance), **do not assume a release id**. -First run discovery and branch on the result: - -``` -python -m orchestrator.cli list --json -``` - -The JSON has `resolution`: -- **`none`** → there is NO active release on this machine. Tell the user briefly, then **use the `m_ask_user` prompt tool** (not a free-text question) to offer starting one — see "Starting a release" below. Only run `init` after they choose. -- **`one`** → use that release (`release.release_id`). Proceed. -- **`ambiguous`** (several exist) → present the list from `all`, then **use `m_ask_user`** to let the user pick which release to act on (one option per release id, most-recent first). Do not act until they choose. -- **`explicit`** (you passed `--release` and it matched) → use it. - -Never run `status`/`next`/`approve` against a release id you haven't confirmed exists via `list`. - -## Starting a release (don't make the user type a date format) - -The release id is just `YYYY-MM`. **You compute it — never ask the user to type the format.** -Work out the current month from today's date (e.g. today 2026-07 → `2026-07`). - -When no release is active (or the user says "start a release"), call the **`m_ask_user`** prompt tool with clickable options, e.g.: -- **"Current month (``)"** ← recommended -- **"A different month"** - -If they pick the current month, run `init --release ` immediately. -If they pick "a different month", then (and only then) ask which month in a follow-up `m_ask_user` free-text prompt (hint: "e.g. next month, or 2026-08") and convert whatever they say into `YYYY-MM` yourself. Accept natural answers ("this month", "next month", "August") — do the date math for them; don't demand a rigid format. -Default to **dry-run**; only pass `--live` if the user explicitly asks for a live run. - -`init` records the **release owner** (the engineer running it) in the release metadata — resolved from the signed-in `az` user, and reminders are emailed to that address. You can pass a richer profile with `--owner-email`/`--owner-name` (e.g. from `workiq_get_my_profile`), or change it later with `set-owner`. Never hardcode a recipient. - -### Ensure push reminders exist (per release — provisioned at start, torn down at close) - -Right after `init`, make sure the **push-reminder automation** exists for THIS release so reminders reach the user even with Scout closed. It is a **per-release** automation: created when the release starts and removed when it closes (see teardown below). -1. List automations (`m_list_automations`). If one named **"Release push reminders"** already exists AND the registry has it scoped to the current release (`automation list --release --json`), **leave it** — don't duplicate. -2. If it's missing, create it with `m_create_automation`: - - **name:** `Release push reminders` - - **schedule:** `every hour` - - **teamsNotify:** `never` - - **prompt:** from `C:\repos\android-complete\release-agent` run `python -m orchestrator.cli tick --json` (this ADVANCES the active release — running the agent steps that can run, holding at gates/actions — then returns `{message, html, subject, owner_email, owner_name, release}`); if `message` is non-empty and `owner_email` is set, email it via `workiq_send_email` (`to: [owner_email]`, `subject:` the `subject` value, `body:` the `html` value with `isHtml: true` — fall back to the plain `message` with `isHtml: false` only if `html` is empty); if `message` is empty, do nothing. (Recipient comes from `owner_email` — never hardcode. Do **not** use `m_send_teams_message` (bot relay 404s) or the Teams self-chat (delivers silently).) -3. **Register it to this release** so it's tracked and torn down at close: - `python -m orchestrator.cli automation register --id --name "Release push reminders" --release --purpose "hourly advance + phase digest email to owner"` - -Do it silently as part of the start flow (the user already opted into push); don't re-ask each release. (The automation runs `tick` in discovery mode, so it targets the active release automatically.) **Why hourly, not once at 9am:** `tick` is idempotent (advancing is a no-op once holding at a gate, and the digest de-dupes to one email per calendar day), so running it every hour means a run missed while the machine was off — e.g. the 9am tick — is simply picked up by the next tick after the machine is on. A single daily trigger would be skipped for that day. - -### Any automation you provision MUST be registered (for teardown) - -Whenever you create a Scout automation for the orchestrator, immediately record it with `automation register` so nothing gets orphaned: -- **Per-release** (the normal case, e.g. push reminders, a phase watcher) → `--release `. **Removed when that release closes.** -- **Shared / persistent** (rare — only something genuinely meant to outlive every release) → `--shared` (no `--release`). Not torn down at close. Default to per-release unless there's a clear reason. - -At **release close** (status complete, the Release Close phase, or the user asks to "clean up automations"), tear down that release's automations: -1. `python -m orchestrator.cli automation list --release --json` — the automations provisioned for this release. -2. For each entry, delete the real Scout automation with `m_delete_automation` (id from the entry), then `python -m orchestrator.cli automation deregister --id `. -3. Shared automations (if any) are **not** in the release-scoped list, so they survive — leave them. -Confirm with the user before deleting, and report what was removed. - -## Code Complete Date (CCD) & phase scheduling - -Phases are **anchored to the Code Complete Date**, not started on demand. **The CCD is the 2nd Wednesday of the release month — that's the canonical default.** `init` computes it and prints when Phase 0 opens. - -`init` also *reads* the pipeline (ADO 3038 `overrideCodeCompleteDate`) but **does not silently adopt it.** If the pipeline holds a **different in-month date**, that's a **conflict to resolve, not an answer**: the status view shows a *"⚠ Confirm the date"* line and `status --json` sets `ccd_conflict`. When you see a conflict, **ask the user which is the real CCD** via `m_ask_user`, e.g.: -- **"Use the 2nd-Wednesday default (``)"** — then offer to sync the pipeline: run `set-ccd --release --default --reason ""` (preview) → show it → `--confirm` to clear the pipeline override so they match. -- **"Use the pipeline date (``)"** — run `set-ccd --release --date --reason "confirmed CCD is " --confirm` (stores it locally; the pipeline already has it). - -Either resolution clears the conflict. Never pick for the user. - -- **Phase 0 (Pre-flight) opens at CCD‑7** (7 days before CCD). You can `init` any time, but until CCD‑7 the release sits in **`scheduled`** — the engine runs nothing. The status view says *"🗓 Scheduled — Pre‑flight opens `` (in N days). Nothing to do yet."* Relay that plainly; don't try to force it forward. -- When the clock reaches CCD‑7, `next` opens Phase 0 and runs its steps up to the first gate — the normal flow resumes. -- **Testing the clock:** every read/advance command accepts `--as-of YYYY-MM-DD` to simulate a date (dry-run only). Real runs use today. - -**Changing the CCD (real production change).** If the user wants to move the date ("give us more time", "cut early"), use `set-ccd`. This **writes the pipeline override** — so it's gated: run it **without `--confirm` first to show the preview**, present that to the user, get an explicit yes (a `--reason` is always required, for audit), then re-run **with `--confirm`**. The override is month-scoped — the date must be in the release month. Use `--default` to revert to the 2nd-Wednesday default. - -**Skipping/cancelling the release.** Same gated pattern: `skip-release` sets the pipeline `skipRelease` switch (preview → confirm, reason required); `skip-release --clear` re-enables it. This suppresses the monthly trigger — treat it as a real, deliberate action and confirm before `--confirm`. - -**Ongoing conflict detection.** `status`/`resume` re-read the pipeline; if someone sets a differing override later, the same `ccd_conflict` surfaces — ask again. (Use `--no-pipeline-check` only if offline.) - -## Push reminders — the daily phase digest (reaching the user when Scout is closed) - -Everything above is **pull** (seen only when the user opens Scout). The **push** layer is a **daily phase status digest** emailed to the release owner, with a deliberate model: - -- **Setup is interactive — no push.** The readiness checklist and establishing the CCD happen live in Scout, so they are **never** emailed. An unsigned release, a blocked entry gate, and a halted release all stay silent. -- **The first push is a phase opening.** Phase 0 opens at **CCD‑7** — that's the first email. Nothing is sent before a phase opens (no pre‑open heads‑up). -- **Daily while a phase has outstanding work.** Once a phase is open, the owner gets a **once‑per‑day** digest (progress + what still needs them) until the phase's actions are done; then the next phase's digest takes over when it opens (each phase notifies on open). - -`tick` is the deterministic automation half: `python -m orchestrator.cli tick --json` first **advances** the active release (runs the agent steps that can run, holding at gates/actions — idempotent), then returns `{message, html, subject, owner_email, owner_name, release}` — `message` is the plain-text digest, `html` is the rich HTML version (full task table with status pills, attention-flagged), both empty when nothing is due today or it was already sent today. (`notify --json` is the read-only variant — same payload but does NOT advance; use it for a manual "what would I be told" check.) `--as-of ` is a debug clock; `--force` bypasses the once‑per‑day guard. - -- A **Scout automation** named **"Release push reminders"** runs `tick --json` (discovery mode) **hourly** and, when `message` is non‑empty, emails it via `workiq_send_email` to `owner_email` (subject from the JSON) — the release owner from release metadata, **never a hardcoded address**; when `message` is empty it stays silent. Running hourly (not once/day) means a tick missed while the machine was off is picked up by the next one, and idempotency + once‑per‑day de‑dup keep it to one advance-effect and one email per day. It is **per‑release**: auto‑provisioned (create‑if‑missing, registered to the release) at start and torn down at close. (Email is the channel because it reliably notifies; the `m_send_teams_message` bot relay 404s without a conversation reference, and the Teams self‑chat delivers silently.) - -If the user asks "how will I be reminded" / "set up notifications," explain this; if the automation doesn't exist, create it (see "Ensure push reminders exist"). Keep the email subject/body exactly as `tick` returns — don't embellish. - -## The readiness ENTRY GATE (right after starting) - -Immediately after `init`, the very first thing is the **readiness checklist** — the entry gate. The engine's `next` refuses to run any step (reports `readiness_gate`) until it's cleared. **Every item is equally required** — there is no priority or "hard vs soft" distinction. The only difference between items is **who resolves them**: - -- **`auto`** — **Scout resolves it** (verifies programmatically, pass/fail). Two execution sources, but the user sees both as `[auto]`: - - *Python-verified* (default): `build_access` (both ADO build definitions, via `az`) and `mcp_servers` (the ICM + Kusto/ADX MCP servers are registered in Scout). The engine's `verify`/`sign` runs these. - - *Scout-assisted* (`source: scout`): the **engine can't reach the MCP/Scout-settings, so YOU run the check** and record the result (see step 3a). Fail-closed **except `silent_perms`** (see below). Today: `oncall_now` (ICM current on-call), `adx_access` (Kusto `print 1` against the ADX cluster), `silent_perms` (Scout permissions allow fully-unattended runs). -- **`attest`** — **the engineer resolves it** (confirms): `play_console_access`, `oncall_window`, `saw_ame`, `yubikey`. - -**Two of the auto items exist so scheduled work runs UNATTENDED** (machine on, Scout not focused): `mcp_servers` (the MCP deps are registered — **hard**, since without them the on-call/telemetry checks can't run) and `silent_perms` (permissions won't stall the daily digest / Teams reminders / browser checks on a prompt — **soft/opt-out**: the user can choose to proceed without silent runs, recorded as `degraded`, with the downside noted). Enabling silent runs needs the user to flip ONE Scout master toggle first (*"Allow AI to request permission changes"*) — only they can (it's read-only from the model); after that I auto-request the rest with a single Allow click. - -**On-call is TWO items (hybrid), because Scout can only see the *current* rotation, not the future one:** -- `oncall_now` (**auto/ICM**) — are you on-call *right now*? Scout verifies this from ICM. -- `oncall_window` (**attest**) — are you free across the whole release window **CCD‑7 → CCD+14**? Scout can't read the future rotation, so you attest it (the checklist shows the concrete dates). - -If any item is unsatisfied the gate stays closed. If the engineer can't satisfy an attest item, they resolve it or hand the release to someone who can — the same for every item. Never describe any item as "not a hard block" or "optional." +You are the conversation layer over the **Release Orchestrator engine** (deterministic Python). The engine decides what happens next; you discover releases, present status/gates, and relay decisions. **Never decide the release flow yourself, and never invent a release — always call the engine.** -Flow after starting: -1. `python -m orchestrator.cli checklist --release --verify` — this runs the auto checks AND prints the **canonical checklist table (markdown)**. **Reproduce its stdout into your reply as live markdown (NOT wrapped in a ``` code fence)** so Scout renders it as a real table — it is already a finished markdown table with the type labels, per-item status, and clickable links. **Do NOT rebuild, re-format, re-order, re-label, or re-type any of it from memory, and do NOT fence it.** If you reconstruct it you WILL introduce errors (stale icons, mangled/merged URLs); if you fence it, it shows as raw text. Always reproduce the literal command output as rendered markdown. You may add a sentence of your own before or after, but the table block itself must match the output. -2. There are exactly **two types by resolver**: `[auto]` (Scout verifies) and `[attest]` (the user confirms). All items must be satisfied to clear the gate. (Do not add lock icons or a "hard requirement" legend.) -3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. **Do this quietly: run the checks and `record-check` them without narrating each one** (a wall of per-check prose is what buries the table). The only time 3a surfaces anything to the user is the `silent_perms` opt-out choice below. - - **`oncall_now` (ICM):** call the ICM MCP `get_on_call_schedule_by_team_id` with `teamIds: [78848]` ("Auth Client Android Shield"). Resolve the current user's alias (`get_my_icm_context` or the owner email's local part), then decide by their role in `shiftCurrentOnCalls[].currentOnCallContacts[]`: - - **Not in the roster at all** → `record-check --item oncall_now --status pass --detail "not on the current roster"`. - - **Present but NOT the primary** (i.e. they are a **backup/secondary** — any position other than the first-listed contact) → **pass**: `record-check --item oncall_now --status pass --detail "backup OCE, not primary (primary: )"`. A backup is free to run the release. - - **The PRIMARY / current OCE** (the **first-listed** contact in `currentOnCallContacts`) → `record-check --item oncall_now --status fail --detail "currently the primary on-call for Auth Client Android Shield"`. - - Only the **primary** blocks the gate. If you cannot confidently tell primary from backup (ambiguous ordering, or the user says otherwise), **ask the user** "Are you the primary/current OCE, or backup?" and record accordingly — **never block a backup.** - - **`adx_access` (Kusto):** run a trivial query — `kusto_query` with the item's `cluster_uri` + `database` (from `checklist --json`) and query `print 1`. Success = the engineer has data access to the ADX release dashboard's cluster. - - Query succeeds → `record-check --release --item adx_access --status pass --detail "print 1 succeeded"`. - - Query fails (auth/access error) → `record-check --release --item adx_access --status fail --detail ""`. - - **`silent_perms` (Scout settings — OPT-OUT/soft):** the daily push digest, the Teams reminders and the browser (CCOA/lockdown) checks all run from a background automation while Scout isn't focused — they must not stall on a permission prompt. Call **`m_get_settings`** and read `permissions.servers`. It's satisfied when ALL of the item's `required_servers` (from `checklist --json`: `shell`, `workiq`, `playwright`) have `autoApprove: true` (one server flag each keeps it simple: `workiq.autoApprove` covers both `workiq_send_email` and Teams; `playwright.autoApprove` covers the browser). **This item never hard-blocks — the user may choose to proceed without silent runs.** Flow: - - **Already all auto-approved** → `record-check --release --item silent_perms --status pass --detail "shell/workiq/playwright auto-approved"`. Done. - - **One or more NOT auto-approved** → **offer the choice** with `m_ask_user`: **"Enable silent runs (recommended)"** vs **"Proceed without — I'll get prompts"**. Explain the downside of proceeding: *the daily digest, Teams reminders, and CCOA/lockdown browser checks will pop a permission prompt when Scout isn't focused and can stall until you open Scout and approve them.* - - They pick **Enable** → the only manual step is the Scout master toggle: if `permissions.allowModelPermissionsChange` is `false`, tell them to turn on **Settings → Permissions → "Allow AI to request permission changes"** (I cannot flip it — it's read-only from the model, by design). Once it's `true`, call **`m_request_permission_escalation`** with `servers: { workiq: {autoApprove:true}, playwright: {autoApprove:true} }` (add `shell` if off too); they click **Allow** once, then re-read `m_get_settings` and `record-check … --status pass --detail "enabled silent runs"`. - - They pick **Proceed without** (or won't enable the master toggle) → `record-check --release --item silent_perms --status degraded --detail "proceeding without silent runs — unattended digest/Teams/browser checks will prompt & may stall until Scout is opened"`. **`degraded` satisfies the gate** (the checklist shows it as ⚠️ *Proceeding (not silent)*), so the release can start; the downside is on record. - - A `fail` on `oncall_now`/`adx_access`, or a real problem, keeps the gate closed — treat it like any unsatisfiable required item (resolve or hand off). Do NOT attest these — they're `auto` items you verified. (`silent_perms` is the one soft/opt-out auto item: it uses `degraded`, never `fail`, when the user chooses to proceed.) -3b. **Re-anchor on the TABLE, then attest.** After recording the auto results, **run `python -m orchestrator.cli checklist --release ` again and reproduce its updated markdown table** — the auto items now show ✅ and only the attests are Outstanding. This table is the single source of truth and **must be shown immediately before you ask for attestations**; the auto-check work in 3a pushes the first table far up, so re-render it here every time. Then use `m_ask_user` to collect the four attestations (`play_console_access`; the on-call **window** `oncall_window` — the CCD‑7 → CCD+14 dates are in the table; `saw_ame`; `yubikey`). Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. **Never replace the table with a plain prose list of the items** — the `m_ask_user` prompt accompanies the re-rendered table, it does not substitute for it. -4. If they confirm everything → `sign --release --all`, then `next` to begin Phase 0. -5. **If they can't satisfy any attest item** (on-call during the window, no SAW, no YubiKey, no portal access) → `decline --release --item ` (repeat `--item` for each). The gate is now blocked. Tell them plainly: the release can't start until that item is resolved; if they can't resolve it, hand the release to another engineer who can (notify their manager / the release team). Treat every item this way — don't single any out as harder or softer. -6. If an **auto** item shows FAIL (no build-definition access, or you're on-call), the gate stays closed — a real problem to resolve, not something to attest around. - -Never attest an `auto` item on the user's behalf — auto items are only satisfied by real verification (Python check or your recorded ICM result). -Never hand-edit or regenerate the checklist/status blocks — always show the CLI's literal output. - -## Parallel phases — process ALL the holds, not one at a time - -Some phases run **in parallel** (Phase 0 is `execution: parallel`): a single `next` runs **every independent automated step at once** (breaking, CG, cron, wiki — all complete in one call) and then surfaces **all the human/scout holds together** (e.g. *"4 item(s) need you: …"*). So don't treat it as one-step-at-a-time. After `next`, read `status --json` and look at **`pending_human`** (and `active_phase.steps` with their `status`/`needs_owner`) — that's the full set of what's outstanding. Work through **all** of them in this pass: -- **`source: scout`** steps (notice, flight_reminder, lockdown) → run each via MCP/browser + `record-step` (see the sections below). These are independent — do them all. - -> **State writes are safe to parallelize.** The CLI serializes every state read-modify-write per release with an exclusive lock, so firing several `record-step` / `record-check` / `done` calls at once (or an hourly `tick` overlapping your command) can't clobber each other — a second invocation simply waits for the first to save. You don't need to run them one-at-a-time to avoid races. -- **`attest`** steps (confirm_reminders, vitals) → ask the owner to confirm, then `done --step `. -- **`blocked`** steps (cg/cron on a real problem) → show the note; fix + rerun, or skip. -Dependencies still hold: `confirm_reminders` only appears **after** `flight_reminder` is sent (it won't be in `pending_human` until then). Call `next` again after clearing holds to let newly-ready steps surface and, once all are done, advance to the next phase. - -> **ALWAYS show the `status` table each iteration — never a bare prose list.** After every `next`, and after every `done`/`record-step` that clears a hold, run `python -m orchestrator.cli status --release ` (no `--json`, add `--as-of` if simulating) and **reproduce its full output as live markdown**: the phase map, the current-phase **steps table** (what's done ✅ / pending / needs you 📌), and the **Results & activity** section (each agent's result — the CG report, the created **wiki link**, block reasons). This is the source of truth the user asked for; the human-readable `status` also auto-logs what was shown. Use `status --json` only for your own branching — but still show the rendered table to the user. Then, alongside the table, tell the user exactly what each outstanding step needs (run the scout steps yourself; for the `attest` steps spell out what to confirm). Do **not** replace the table with your own summarized list of remaining steps. - -## Scout-assisted phase steps (CCOA lockdown check) - -Some Phase steps read AAD-gated sources the deterministic engine can't reach, so **you run them via the browser and record the result** — same idea as the readiness scout checks, but mid-phase. When advancing, if **`lockdown`** is among the pending holds (`source: scout`, in `pending_human`), handle it like this — silently, without bothering the user unless there's an overlap: - -1. **Scrape the CCOA source.** Navigate (Playwright) to `https://prod.change-manager.msidentity.com/ccoa-periods`. If an AAD account picker appears, click the user's own account (Windows-SSO — no password). Wait for the "CCOA Periods" page. -2. **Extract the periods.** From **"Upcoming CCOA periods"** and the **current-year** "Past NoFly Zones" table, read each row's **Name, Environment, Start Date (UTC), End Date (UTC)**. Build a JSON array: `[{"name","environment","start":"YYYY-MM-DD","end":"YYYY-MM-DD"}, ...]` (use the UTC dates). -3. **Let the engine decide (deterministic).** Run `python -m orchestrator.cli check-lockdown --release --periods-json ''`. It computes the release window (CCD‑7 … CCD+14), keeps only **Production**-environment periods, checks overlap, and records the step: **pass** (no overlap → step done, flow continues) or **attention** (overlap → step holds). -4. **Relay the outcome.** On **pass**, just continue (`next`) — no need to bother the user. On **attention**, surface it: name the overlapping lockdown(s) and window, and tell them to **shift CCD** past the lockdown (`set-ccd`) if they want to proceed; there are **no partners to notify** for this step. - -If you can't reach the browser/SSO in this context, leave the step held — it stays flagged as needing attention and you (or the user, next time Scout is open) can run it then. Don't mark it done without actually running the check. - -## Scout-assisted phase steps (early code-complete notice) - -The Phase-0 `notice` step sends the early code-complete email. Sending needs WorkIQ (a skill capability), so it's scout-assisted like `lockdown`. When `status --json` shows the current step is **`notice`** (holding, `awaiting_action`): - -1. **Prepare it (deterministic).** Run `python -m orchestrator.cli prepare-notice --release `. It fills the local template (`templates/early-code-complete-notice.md`) with the release's CCD/owner and returns JSON `{subject, body, html, recipients, dry_run, recipients_note}`. -2. **Send it.** Email via `workiq_send_email` using the returned `subject` and `recipients` exactly, with **`body:` the `html` value and `isHtml: true`** (the HTML has a clean hotfix-guide link + a proper rendered table — fall back to the plain `body` with `isHtml: false` only if `html` is empty). **Recipients are already resolved for you**: in a **dry-run** they're the **release owner only** (safe rehearsal — the subject is prefixed `[DRY-RUN → owner]`); on a **live** release they're the real distribution list (androididentity@microsoft.com, jialh@microsoft.com — see EXTERNAL-REFERENCES.md). Never override the recipients. -3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step notice --status pass --detail "sent to "`. If the send fails, `--status attention --detail ""` to keep it flagged. - -## Scout-assisted phase steps (flight & string reminders — Teams) - -The Phase-0 `flight_reminder` step posts a **combined 4-in-1 reminder** (update local flights · flight pre-mortem docs · merge user-facing strings by CCD-7 · Auth App feature-flag freeze / default-OFF review) as a **Teams message** to the Android Core Team. Sending Teams needs WorkIQ, so it's scout-assisted. When the current step is **`flight_reminder`** (holding): - -1. **Prepare it.** Run `python -m orchestrator.cli prepare-flight-reminder --release `. It returns JSON `{content, content_type:"html", dry_run, send_to, owner_email, chat_id, target_note}`. -2. **Resolve the chat + send.** - - **Dry-run** (`send_to: "owner"`): get the owner's 1:1 chat with `workiq_create_chat_by_email` (email = `owner_email`), then `workiq_send_chat_message` with that `chatId`, `content` = the returned HTML, `contentType: "html"`. (Safe rehearsal — the message is prefixed `[DRY-RUN → owner]`.) - - **Live** (`send_to: "group"`): `workiq_send_chat_message` with `chatId` = the returned `chat_id` (the Android Core Team thread), `content`, `contentType: "html"`. - Never override the target — `prepare-flight-reminder` already picked owner-vs-group from dry_run. -3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step flight_reminder --status pass --detail "posted to "`; on failure, `--status attention --detail ""`. - -**Sending the reminder is fire-and-forget** — it does NOT prove the feature owners actually did the work. So the very next step is **`confirm_reminders`**, a human **attestation** the engine holds on (`awaiting_action`). When the current step is `confirm_reminders`, ask the release owner (via `m_ask_user`) to confirm the reminded work is actually done — feature owners updated local flights, wrote flight pre-mortem docs, merged user-facing strings by CCD-7, and all features are default-OFF (or default-ON ones are approved in the wiki). Only when they confirm, run `python -m orchestrator.cli done --release --step confirm_reminders --note ""`. If they can't confirm, leave it holding (the release correctly blocks here until the pre-requisite work is verified) — don't mark it done. - -Phase 0's **`vitals`** step ("Confirm Play Console vitals & policy status reviewed") is another **attestation** hold. Play Console has no API for **Policy issues/warnings** (the Reporting API covers only technical vitals, and the Console UI is behind a Google login Scout can't automate), so this is a manual check: when the current step is `vitals`, ask the owner to open Play Console, review **Android vitals** (crash/ANR rate) and **Policy status** (issues/warnings), and confirm they're acceptable. On confirmation, `python -m orchestrator.cli done --release --step vitals --note ""`. If there's an unresolved policy issue or vitals regression, leave it holding. - -## Commands (run from the release-agent folder) - -| Intent | Command | +## Where things live +- Engine + config: `C:\repos\android-complete\release-agent\` — **run all `python -m orchestrator.cli …` commands from here.** +- Run-state: `C:\repos\android-complete\.release-runs\\release-state.json` (gitignored; one per month, e.g. `2026-08`). +- **Reference docs (this skill's detail):** `C:\repos\android-complete\release-agent\skill\reference\` — read the relevant one on demand (routing table below). The core stays lean; the details live there. +- `setup/bootstrap.ps1` only prepares the machine (infra preflight + installs this skill). If an infra check fails (an MCP server isn't registered, or Scout wasn't restarted), run `python -m orchestrator.cli infra` and tell the user to restart Scout; manifest is `config/requirements.yaml`. + +## GOLDEN RULES (always apply — the deduped essentials) +1. **Discover first, always.** On ANY release request, run `python -m orchestrator.cli list --json` and branch on `resolution`: `none` → offer to start (via `m_ask_user`); `one` → use `release.release_id`; `ambiguous` → list `all`, let the user pick; `explicit` → use it. Never run `status`/`next`/`approve` against an unconfirmed id. +2. **Render CLI output as LIVE MARKDOWN — never fenced.** `checklist`, `status`, `next`, etc. print finished markdown tables. Reproduce their stdout **verbatim as normal message content** so Scout renders the table — do NOT wrap in a ``` code fence, and do NOT rebuild/re-order/re-type from memory (you'll introduce stale icons / broken URLs). A sentence before/after is fine; the block must match. Use `--json` only for your own branching. +3. **The engine is the source of truth.** It owns sequencing and gate state. When unsure, `status --json`. Never hand-edit checklist/status output. +4. **Prompt, don't interrogate.** For any discrete choice (start? which release? approve/deny?) use the `m_ask_user` clickable prompt, not free-text. Reserve free-text for genuinely open values (an unusual month). +5. **Never assume a human decision.** An `m_ask_user` result that merely echoes the offered options is NOT confirmation. Never attest, approve, sign, or mark done until the user explicitly said so. Attesting/approving on an assumption is a release-integrity violation. +6. **Gates are human-decided.** Present and relay Approve/Deny; never authorize yourself. +7. **Dry-run by default.** Only pass `--live` when the user explicitly asks. `[STUB…]` output = mocked step; say so, don't imply real work. +8. **Never hardcode a recipient.** Reminders/notices go to the release `owner_email` from metadata (or engine-resolved DLs). +9. **Log silently.** Human-readable commands auto-log. YOU must journal user choices: `journal --release --source user --kind choice --text "" --choice "
    • " if len(names) < s["failed"] else "") + tag = _chip(_CAT_LABEL.get(s.get("category", "ui"), "UI automation"), "#eef4ff", "#0b5cad") suite_html += ( - f"
    " - f"
    {s['failed']}" - f" / {s['total']}  {T.esc(s['name'])}
    " - f"
      " - f"{items}{more}
    ") - return (f"
    " - f"
    MRWP {prov} — run {r.get('run_id')} " - f"({r.get('ran')}/{r.get('total')} stages)
    " - f"
    Tests: {tline}
    {red}{suite_html}" - f"
    ") + f"
    " + f"" + f"" + f"
    {T.esc(s['name'])}  {tag}" + f"{s['failed']}" + f"/{s['total']} " + f"· {sr}%
    " + f"
      {items}{more}
    ") + + rate_color = "#b42318" if ui_rate >= 5 else ("#b54708" if ui_rate > 0 else "#067647") + return ( + f"" + f"
    " + f"" + f"" + f"
    MRWP {prov}" + f" · run {r.get('run_id')} " + f"· {r.get('ran')}/{r.get('total')} stages{_chip('completed', '#ecfdf3', '#067647')}
    " + # headline = UI-automation failure rate (the RC-critical bucket) + f"
    " + f"{ui_rate}%" + f" UI-automation failure rate  ·  " + f"{ui_pass} passed / " + f"{ui_fail} failed of {ui_total} UI tests
    " + f"{_split_bar(100 - ui_rate)}" + # per-category breakdown + f"{cat_table}
    " + f"{red}{suite_html}" + f"" + f"
    ") + + # Overall headline — UI-automation failures ONLY (the RC-critical bucket), across both providers. + def _ui_sum(field): + return sum((((model.get("mrwp") or {}).get(p) or {}).get("tests", {}) + .get("categories", {}).get("ui", {}).get(field, 0) or 0) + for p in ("ECS", "Local")) + tot_f, tot_t = _ui_sum("failed"), _ui_sum("total") + overall_rate = _fail_rate(tot_f, tot_t) probs = model.get("problems") or [] issues = (("
    Blocking issues (a stage that never " + "border-radius:8px;color:#b42318;'>Blocking issues (a stage that never " "ran = pipeline aborted):
      " + "".join(f"
    • {T.esc(p)}
      • " for p in probs) + "
    ") if probs else "") return f"""\ -
    -

    Hi {T.esc(ctx.get('owner','there'))},

    -

    The Release Candidate for {T.esc(rid)} has been built and RC testing has completed. - Review the results below and approve “RC verified — proceed to bug bash” when ready.

    -

    Pipeline health

    -
      -
    • Code Complete Checker: fired the release (run {ch.get('run_id')}).
      • -
    • Release Orchestrator: healthy — pre-gate stages green, {park}.
      - Versions: {T.esc(vstr)} · - run {o.get('run_id')}
      • -
    -

    RC testing — both provider runs ran to completion

    - {mrwp_block('ECS')} - {mrwp_block('Local')} +
    + + +
    +
    RC Verification Report
    +
    Release {T.esc(rid)} · Phase 2 — Build & RC testing
    +
    + + + + + + + +
    +
    UI-automation failure rate
    +
    {overall_rate}%
    +
    {tot_f} failed / {tot_t} UI tests
    +     		+
    Checker
    +
    ✓ Fired
    +
    run {ch.get('run_id')}
    +     		+
    Orchestrator
    +
    ✓ Healthy
    +
    {park}
    +
    + +

    Versions: {T.esc(vstr)} · + orchestrator run {o.get('run_id')}

    + +

    UI-automation results

    + {mrwp_card('ECS')} + {mrwp_card('Local')} {issues} -

    Next: if the failures are acceptable to carry into bug bash, approve the gate - (the release advances to Phase 3 — Test / Bug Bash). Otherwise investigate the red suites first.

    -

    — Release Orchestrator (Scout)

    + + + +
    + Next: review the failing suites above. If acceptable to carry into bug bash, + approve “RC verified — proceed to bug bash” + (advances to Phase 3). Otherwise investigate the red suites first. +
    +

    — Release Orchestrator (Scout)

    """ diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 0cfa4996..8f108857 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2005,10 +2005,18 @@ def test_build_verify_rc_report_emails_owner(): "versions": {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}}, "mrwp": {"ECS": {"run_id": 1678863, "complete": True, "ran": 23, "total": 23, "failed_stages": ["UI Automation"], - "tests": {"total": 5871, "passed": 5767, "failed": 104, - "runs": [{"name": "PROD MSAL - RC Broker", "total": 44, "failed": 18}]}}, + "tests": {"total": 5871, "passed": 5767, "failed": 104, "runs": [], + "categories": { + "unit": {"total": 5248, "passed": 5248, "failed": 0}, + "instrumented": {"total": 442, "passed": 440, "failed": 2}, + "ui": {"total": 165, "passed": 63, "failed": 102}}}, + "failed_suites": [{"name": "PROD MSAL - RC Broker (API 32)", + "failed": 18, "total": 44, "category": "ui", + "tests": ["test_1_Foo", "test_2_Bar"]}]}, "Local": {"run_id": 1678864, "complete": True, "ran": 23, "total": 23, - "failed_stages": [], "tests": {"total": 5856, "passed": 5756, "failed": 100, "runs": []}}}, + "failed_stages": [], "tests": {"total": 5856, "passed": 5756, "failed": 100, + "runs": [], "categories": {}}, + "failed_suites": []}}, "problems": []} try: st = ReleaseState(release_id="2026-08", ccd="2026-08-26", @@ -2016,7 +2024,12 @@ def test_build_verify_rc_report_emails_owner(): out = as_dict(_steps.get_step("build_verify", "rc_report").build(st)) assert out["kind"] == "needs_skill" and out["tool"] == "workiq_send_email" assert out["payload"]["to"] == ["dev@microsoft.com"] and out["payload"]["isHtml"] - assert "104 failed" in out["payload"]["body"] and "1678863" in out["payload"]["body"] + body = out["payload"]["body"] + assert "1678863" in body # run id present + assert "UI-automation failure rate" in body # per-category headline metric + assert "61.8%" in body # 102/165 UI failures — the real UI rate + assert "Unit" in body and "Instrumented" in body and "UI automation" in body + assert "test_1_Foo" in body # failing test names still listed assert out["record_as"] == "rc_report" and out["outbound"] is True # no owner → blocked st2 = ReleaseState(release_id="2026-08", ccd="2026-08-26") @@ -2026,6 +2039,18 @@ def test_build_verify_rc_report_emails_owner(): P.release_report = orig +def test_classify_test_run_categories(): + """The test-run classifier buckets into exactly three: unit / instrumented / ui; + anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" + from tools import pipelines as P + assert P.classify_test_run("common4j_UnitTests") == "unit" + assert P.classify_test_run("common_InstrumentedTests") == "instrumented" + assert P.classify_test_run("PROD MSAL - RC Broker (API 32)") == "ui" + assert P.classify_test_run("RC MSAL - PROD Broker (API 28) # 123_build.1") == "ui" + assert P.classify_test_run("Lab Api Tests") == "ui" # NOT 'other' + assert P.classify_test_run("") == "ui" + + def test_get_failed_tests_aggregates_repeated_suites(): """get_failed_tests merges the SAME suite that appears as several runs (the cause of the confusing duplicates) into one entry, summing failures and collecting test names.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 2c2381e9..645f9510 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -295,10 +295,33 @@ def stage_completion(stages): "failed": failed, "yellow": yellow, "complete": not never and total > 0} +import re as _re_mod +_UI_API_RE = _re_mod.compile(r"\(API\s*\d+\)", _re_mod.IGNORECASE) +TEST_CATEGORIES = ("unit", "instrumented", "ui") +_CATEGORY_LABEL = {"unit": "Unit", "instrumented": "Instrumented", "ui": "UI automation"} + + +def classify_test_run(name): + """Bucket a test-run/suite name into one of THREE categories: + * '*_UnitTests' → unit + * '*_InstrumentedTests' → instrumented + * everything else → ui (the device UI-automation suites, which carry an + '(API NN)' tag, plus any other run such as + 'Lab Api Tests' — 'the rest are UI'). + """ + low = (name or "").lower() + if "unittest" in low: + return "unit" + if "instrumentedtest" in low: + return "instrumented" + return "ui" + + def get_test_summary(org, project, build_id, timeout=60): """Return (ok, summary, detail) for a build's Test-tab results. summary = - {total, passed, failed, runs:[{name,total,passed,failed}]} aggregated across all - test runs associated with the build (unit / instrumented / UI-automation). + {total, passed, failed, runs:[{name,total,passed,failed,category}], + categories:{unit|instrumented|ui|other: {total,passed,failed}}} aggregated across + all test runs, classified into unit / instrumented / UI-automation / other. Uses the Test Runs REST API directly (az devops invoke mis-routes this one).""" base = org.rstrip("/") @@ -309,16 +332,22 @@ def get_test_summary(org, project, build_id, timeout=60): return (False, None, detail) runs = (data or {}).get("value", []) or [] out_runs, tot, passed = [], 0, 0 + cats = {c: {"total": 0, "passed": 0, "failed": 0} for c in TEST_CATEGORIES} for r in runs: t = r.get("totalTests") or 0 p = r.get("passedTests") or 0 na = r.get("notApplicableTests") or 0 f = max(t - p - na, 0) + cat = classify_test_run(r.get("name")) tot += t passed += p - out_runs.append({"name": r.get("name"), "total": t, "passed": p, "failed": f}) + cats[cat]["total"] += t + cats[cat]["passed"] += p + cats[cat]["failed"] += f + out_runs.append({"name": r.get("name"), "total": t, "passed": p, + "failed": f, "category": cat}) return (True, {"total": tot, "passed": passed, "failed": max(tot - passed, 0), - "runs": out_runs}, "") + "runs": out_runs, "categories": cats}, "") def _suite_base_name(name): @@ -349,7 +378,8 @@ def fcount(r): suites, calls = {}, 0 for r in failing: name = _suite_base_name(r.get("name")) - s = suites.setdefault(name, {"name": name, "failed": 0, "total": 0, "tests": []}) + s = suites.setdefault(name, {"name": name, "failed": 0, "total": 0, + "category": classify_test_run(name), "tests": []}) s["failed"] += fcount(r) s["total"] += r.get("totalTests") or 0 if calls < max_result_calls: From 2a8c5fc3d4b9759914b93fcd4f1c6ec4899f15a3 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 17:53:13 +0100 Subject: [PATCH 63/90] release-agent: document that _ccd_cron emits host-local wall-clock (scheduler is local, not UTC) Empirically verified 2026-08-20: a cron '37 9' fired at 09:37 PDT / 16:37 UTC. Docstring-only note so a UTC conversion is never added, which would shift every CCD-day comm by the host's UTC offset. No logic change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/automations.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/release-agent/orchestrator/automations.py b/release-agent/orchestrator/automations.py index af04cf7a..fc15de9a 100644 --- a/release-agent/orchestrator/automations.py +++ b/release-agent/orchestrator/automations.py @@ -120,7 +120,13 @@ def _ccd_cron(ccd_date, hhmm: str): a CCD more than a week out (these are provisioned at release start) is the wrong date — it fired the CCD-day comms a week early. Cron `M H D Mo *` targets the CCD's day-of-month + month exactly, so a one-shot fires ON the CCD. Returns the NL Scout - accepts (e.g. 'cron: 0 9 26 8 *') or None if inputs are missing/invalid.""" + accepts (e.g. 'cron: 0 9 26 8 *') or None if inputs are missing/invalid. + + TIMEZONE: emit the LOCAL wall-clock time directly — do NOT convert to UTC. + Scout's scheduler interprets cron in host-local time (empirically verified + 2026-08-20: a cron '37 9' fired at 09:37 PDT / 16:37 UTC, not 09:37 UTC). So a + `hhmm` of '09:00' correctly fires at 09:00 local on the CCD. Adding a UTC + conversion here would shift every CCD-day comm by the host's UTC offset.""" if not ccd_date or not hhmm: return None try: From 545ad67cb45c8a0a2b258912dadfa45819c5c1cd Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 12:11:50 -0700 Subject: [PATCH 64/90] Implement RC report follow-up command and UI gate logic; enhance documentation and tests --- .../orchestrator/commands/rc_report.py | 48 ++++++++ release-agent/orchestrator/engine.py | 1 + release-agent/skill/SKILL.md | 1 + release-agent/skill/reference/commands.md | 3 +- .../skill/reference/phases/build_verify.md | 47 ++++++++ .../reference/starting-and-scheduling.md | 2 +- release-agent/steps/build_verify/_common.py | 77 +++++++++++- release-agent/steps/build_verify/rc_report.py | 39 ++++-- release-agent/tests/test_engine.py | 113 ++++++++++++++++++ 9 files changed, 317 insertions(+), 14 deletions(-) create mode 100644 release-agent/skill/reference/phases/build_verify.md diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 5c370c2a..08473d64 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -52,6 +52,46 @@ def _u(build_id): return K.build_url(build_id) if build_id else "" +def cmd_record_rc_report(args): + """Record the rc_report step's outcome AFTER the skill has emailed the RC report. + + Re-reads the live model, applies the UI-automation quality gate (K.rc_ui_gate), + records `pass` (>=90% UI pass → step done, flow advances to go_test) or `attention` + (<90% → step BLOCKS for owner investigation), and stashes the evaluated pipeline-run + links on the step so its Details point at every artifact behind the verdict. + + This is the follow-up the rc_report NeedsSkill names (`payload.followup_command`), so + the skill runs it instead of a blind `record-step --status pass`.""" + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + try: + model = K.rc_report_model(orch.state) + except Exception as e: # pragma: no cover - defensive + print(_json.dumps({"error": f"could not build the RC model ({e})."})) + return 1 + + gate = K.rc_ui_gate(model) + links = K.rc_run_links(model) + status = "pass" if gate["verdict"] == "pass" else "attention" + orch.record_scout_step("build_verify", "rc_report", status, gate["detail"]) + + # record_scout_step doesn't carry links — attach the evaluated-run refs (and stamp + # the recorder as scout) on the resulting step, preserving its status/note. + step = orch.state.get_step("build_verify", "rc_report") + step.links = links + step.by = "scout" + orch.state.set_step("build_verify", "rc_report", step) + C.save_state(orch.state, args.runs_root, args.release) + + C.emit(args.runs_root, args.release, + f"[{'ok' if status == 'pass' else 'attention'}] rc_report: " + f"{gate['detail'].splitlines()[0]}", kind="step") + print(_json.dumps({"verdict": gate["verdict"], "status": status, + "pass_pct": gate["pass_pct"], "ui_total": gate["ui_total"], + "ui_failed": gate["ui_failed"], "threshold": gate["threshold"], + "detail": gate["detail"], "links": links})) + return 0 if status == "pass" else 2 + + def _format(m) -> str: L = [f"## RC Pipeline Status — Release {m['release']}", ""] @@ -138,3 +178,11 @@ def register(sub): rp.add_argument("--release", required=True) rp.add_argument("--json", action="store_true", help="Emit the raw report model") rp.set_defaults(func=cmd_rc_report) + + rr = sub.add_parser( + "record-rc-report", + help="Record the rc_report step after emailing: apply the 90%% UI gate " + "(pass|attention/block) + stash the evaluated run links") + rr.add_argument("--release", required=True) + rr.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + rr.set_defaults(func=cmd_record_rc_report) diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 2b6ea2bd..567e0748 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -648,6 +648,7 @@ def _active_phase_report(self) -> Optional[dict]: "needs_owner": needs, "time_ready": self._step_time_ready(phase, s), # False = waits for its fire_at_local "note": stp.note, # agent result / block reason / detail + "links": list(getattr(stp, "links", None) or []), # durable refs to items evaluated "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_attest or s_blocked)), }) opens = self._phase_anchor_date(phase) diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index a32bcef8..56bb23f9 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -55,6 +55,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → | Running the readiness entry gate (right after `init`) | `reference/readiness-gate.md` | | Starting a release / handling CCD / setting up push reminders & automations | `reference/starting-and-scheduling.md` | | Advancing **Phase 0 (Pre-flight)** — notice, flight reminders, lockdown, confirm, vitals | `reference/phases/preflight.md` | +| Advancing **Phase 2 (Build & RC Verification)** — verification chain, RC report email + 90% UI gate, go_test | `reference/phases/build_verify.md` | | Rendering `status`/`checklist` output | `reference/presenting-status.md` | | Looking up a command / manual override / event-logging detail | `reference/commands.md` | | Building a NEW phase's guidance | `reference/phases/_TEMPLATE.md` | diff --git a/release-agent/skill/reference/commands.md b/release-agent/skill/reference/commands.md index 11ce610c..96110fb1 100644 --- a/release-agent/skill/reference/commands.md +++ b/release-agent/skill/reference/commands.md @@ -15,6 +15,7 @@ _Loaded on demand. Run all from `C:\repos\android-complete\release-agent`._ | Resolve a migrated step → outcome JSON (done\|blocked\|needs_human\|needs_skill) | `python -m orchestrator.cli step-action --release --step [--phase

    ] [--param k=v …]` | | Answer a STEP question (knowledge) | `python -m orchestrator.cli step-info --step [--phase

    ]` | | **Phase 2 — RC pipeline + test report** (read-only) | `python -m orchestrator.cli rc-report --release [--json]` → the checker→orchestrator→ECS/Local-MRWP chain + per-run test breakdown | +| **Phase 2 — record RC verdict** (after emailing the report) | `python -m orchestrator.cli record-rc-report --release ` → applies the **90% UI-automation gate** across both MRWP runs, records `pass` (step done → go_test) or `attention` (step **blocks** for investigation), and stashes the checker/orchestrator/ECS/Local run links on the step. This is the follow-up the `rc_report` `needs_skill` names — run it **instead of** `record-step` | | **Simulate a mid-release point** (testing) | `python -m orchestrator.cli sim list` · `python -m orchestrator.cli sim run --scenario [--freeze] [--json]` → **seeds the real release** to a scenario's target (`config/scenarios/.yaml`): fast-forwards the real engine, signs the entry gate + completes earlier phases from mocks, then stops `open`/`gate`/`done` at the target. `data: live` runs the target phase against real `az`; `data: mock` is offline. Any existing state at that id is backed up first, so afterwards you use the **normal** commands (`status`, `rc-report`, `next`, `approve`). `--runs-root ` targets a throwaway sandbox instead; `--freeze` snapshots state to `tests/fixtures/.json` | | Answer an ENTRY-GATE item question (knowledge) | `python -m orchestrator.cli gate-info --item ` (build_access, mcp_servers, ccd_confirmed, silent_perms, teams_notify, adx_access, oncall_now, play_console_access, oncall_window, saw_ame, yubikey) | | Prepare early code-complete notice (JSON) — _legacy; prefer `step-action --step notice`_ | `python -m orchestrator.cli prepare-notice --release [--variant initial\|update]` | @@ -57,7 +58,7 @@ Map natural language to these ("skip the CG report, doesn't apply" → `skip … - **`done`** — already complete; nothing to run. - **`blocked`** — surface `reason` to the owner; don't proceed. - **`needs_human`** — show `prompt` (attestation or reminder to-do). -- **`needs_skill`** — run `tool` with `payload` (an MCP/browser call the engine can't make, already fully resolved), then confirm with `record-step --step --status pass\|attention`. Runs are real — the payload targets the real DL/chat unless the engineer's `mocks.local.yaml` has a `send_to` redirect (then `payload.to`/`chatId` points at them and the subject carries `[TEST → me]`). +- **`needs_skill`** — run `tool` with `payload` (an MCP/browser call the engine can't make, already fully resolved), then confirm with `record-step --step --status pass\|attention`. **UNLESS `payload.followup_command` is set** — then run that engine command **instead** of `record-step` (it records the verdict itself): `rc_report` sets `followup_command: record-rc-report` (send the email, then run `record-rc-report` — it applies the 90% UI gate and records pass/block + links). Runs are real — the payload targets the real DL/chat unless the engineer's `mocks.local.yaml` has a `send_to` redirect (then `payload.to`/`chatId` points at them and the subject carries `[TEST → me]`). If a step isn't migrated yet, `step-action` returns `{"error": …}` with exit 1. **Use `step-action` for scout steps** (`needs_skill` → run the tool, then `record-step`) **and attest steps** (`needs_human` → show the `prompt` via `m_ask_user`, then clear with `done --step `). Migrated: scout — `preflight.notice`, `preflight.flight_reminder`, `preflight.lockdown` (gather-then-decide: its `needs_skill` carries a `_gather` browser-scrape directive + a `check-lockdown` follow-up); attest — `preflight.confirm_reminders`, `preflight.vitals`. **Agent steps** (`preflight.breaking`, `cg`, `cron`, `wiki`) are migrated too but the **engine runs them in-process during `next`** — `step-action` refuses them (exit 1); relay their results from the `status` table. diff --git a/release-agent/skill/reference/phases/build_verify.md b/release-agent/skill/reference/phases/build_verify.md new file mode 100644 index 00000000..e6163f38 --- /dev/null +++ b/release-agent/skill/reference/phases/build_verify.md @@ -0,0 +1,47 @@ +# Reference — Phase `build_verify` (Phase 2 · Build & Lib Verification) + +Opens **CCD+1** — the engineer wakes to a resume. The engine runs the four verification +**agent** steps in-process during `next`; you relay their results and drive the one +**scout** step (`rc_report`) + the human **gate** (`go_test`). + +## Execution model +Sequential. A single `next` runs the agent chain (checker → orchestrator → ECS/Local +MRWP); each **blocks** on a real problem (a stage that never ran, an unhealthy +orchestrator, an auth failure). A blocked step → show the note, then **fix + `next`** to +re-check, or **`skip … --reason`** to override. When the chain is green the scout +`rc_report` step becomes ready, then the `go_test` gate. + +## Automated steps (no skill action — relay from the `status` table) +`checker_fired`, `orchestrator_health`, `mrwp_ecs`, `mrwp_local` — read-only `az` agent +steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 link. +`step-action` refuses them (exit 1); never dispatch them yourself. + +## `rc_report` — email the RC report + apply the 90% UI gate (`scout`) +- **Trigger:** `status --json` shows current step `rc_report` (state `scout`), after the + four agent steps are done. +- **Resolve:** `step-action --release --phase build_verify --step rc_report` → + `needs_skill` (`workiq_send_email`) with a fully-composed HTML dashboard, plus + `payload.followup_command: record-rc-report`. +- **Act:** send the email verbatim (`payload.to/subject/body`, `isHtml:true`) — honoring + a `send_to` redirect if the engineer set one. **Always send** — the owner gets the + dashboard (failing suites + run links) whether the gate passes or not. +- **Record (the two-hop):** because `followup_command` is set, run + `record-rc-report --release ` **instead of** `record-step`. It re-reads the model, + applies the **90% UI-automation gate** (combined pass rate across ECS + Local): + - **≥ 90% → `pass`** — the step is done; advance to `go_test`. + - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). This is a large UI + failure: the owner must **investigate the root cause** (usually a **fix + an MRWP + re-run**). Exits: fix + re-run, then `next` re-runs `rc_report`; or `skip … --reason` + to override. It records the failing-suite summary + stashes the checker/orchestrator/ + ECS/Local run links on the step. + - (No UI tests found → passes with a ⚠ note.) +- The command prints `{verdict, pass_pct, ui_total, detail, links}` for your branching; + relay the `status` table (the `rc_report` Details shows the verdict + 🔗 links). + +## `go_test` — RC verified, proceed to bug bash (`gate`, human) +Present the settled `status`; `m_ask_user` Approve/Deny; `approve` / `deny --comment`. +Never authorize yourself. If `rc_report` blocked, `go_test` isn't reached until it clears. + +## External references +Engineering pipelines: Checker def 3038, Orchestrator def 2828, MRWP def 2519 +(org `identitydivision.visualstudio.com`, project `Engineering`). diff --git a/release-agent/skill/reference/starting-and-scheduling.md b/release-agent/skill/reference/starting-and-scheduling.md index 453ae6c1..1389eb56 100644 --- a/release-agent/skill/reference/starting-and-scheduling.md +++ b/release-agent/skill/reference/starting-and-scheduling.md @@ -24,7 +24,7 @@ Right after `init`, make sure the **push-reminder automation** exists for THIS r - **teamsNotify:** `never` - **prompt:** From `C:\repos\android-complete\release-agent`, advance the active release **AUTONOMOUSLY** (no user is watching) and send the daily digest: 1. Run `python -m orchestrator.cli status --json`. If there is **no** release, or it is **unsigned / halted / complete**, STOP silently. - 2. **Run Scout's own steps until none remain.** Loop: run `next --json`; read `scout_pending`; if empty, go to step 3; else for EACH id: run `step-action --release --phase --step ` (Phase 0 = `preflight`), perform the returned `needs_skill` `tool`+`payload` via the matching MCP tool (`workiq_send_email` / `workiq_send_chat_message` / `azure_devops-pipelines_run_pipeline`, honoring `test_redirect`), then finalize with `record-step … --status pass` — **UNLESS** the action names a follow-up engine command (e.g. `check-lockdown`, `record-localization-run`/`check-localization`), which you run instead as the action describes. **Headless safety copy:** whenever a `needs_skill` action has **`outbound: true`**, after performing it also `m_send_teams_message` a **ONE-LINE** courtesy copy — `🤖 [release ] Autonomous:

    .` — so the owner sees what went out (never paste the full email/message body). A scout step that records `attention` is left blocked (it surfaces in the digest). Do it silently. + 2. **Run Scout's own steps until none remain.** Loop: run `next --json`; read `scout_pending`; if empty, go to step 3; else for EACH id: run `step-action --release --phase --step ` (Phase 0 = `preflight`), perform the returned `needs_skill` `tool`+`payload` via the matching MCP tool (`workiq_send_email` / `workiq_send_chat_message` / `azure_devops-pipelines_run_pipeline`, honoring `test_redirect`), then finalize with `record-step … --status pass` — **UNLESS** the action names a follow-up engine command (via `payload.followup_command`, or the known two-hop steps: `check-lockdown`, `record-localization-run`/`check-localization`, and **`record-rc-report`** for `rc_report` — send the RC email, then run `record-rc-report`, which applies the 90% UI gate and records pass/block), which you run instead as the action describes. **Headless safety copy:** whenever a `needs_skill` action has **`outbound: true`**, after performing it also `m_send_teams_message` a **ONE-LINE** courtesy copy — `🤖 [release ] Autonomous: .` — so the owner sees what went out (never paste the full email/message body). A scout step that records `attention` is left blocked (it surfaces in the digest). Do it silently. 3. Run `python -m orchestrator.cli tick --json` → `{message, html, subject, owner_email, owner_name, release, channels, teams}`. If `message` is empty, STOP (nothing due / already sent today). Otherwise deliver on every enabled channel: - **Email** (when `channels.email`): `workiq_send_email` (`to:[owner_email]`, `subject:` the value, `body:` the `html` with `isHtml:true` — fall back to plain `message`/`isHtml:false` only if `html` empty). Recipient from `owner_email` — never hardcode. - **Teams** (when `channels.teams` and `teams` is non-null): dispatch on `teams.via` — diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 0c1beb04..7d199c98 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -59,6 +59,12 @@ def stash_runs(state, **ids): state.pipeline_runs = pr +# The Phase-2 quality bar: at least this % of UI-automation tests must pass for RC to +# clear to bug bash. Below it, the rc_report step blocks for owner investigation +# (a large UI failure usually means a real regression → fix + re-run MRWP). +RC_UI_PASS_THRESHOLD = 90.0 + + # ---------------------------------------------------------------- RC report email def rc_report_model(state, timeout=120): """The full Phase-2 RC report model (checker → orchestrator → ECS/Local MRWP + @@ -68,10 +74,77 @@ def rc_report_model(state, timeout=120): checker_def=CHECKER_DEF, orch_def=ORCHESTRATOR_DEF, timeout=timeout) +def rc_run_links(model) -> list: + """Durable links to EVERY pipeline run the RC verification evaluated — the Code + Complete Checker, the Release Orchestrator, and both MRWP (ECS + Local) runs — so the + recorded step points at each artifact behind the verdict (surfaced in the step's + Details). Only runs with a resolved id are included.""" + out = [] + ch = (model.get("checker") or {}).get("run_id") + if ch: + out.append({"name": "Code Complete Checker run", "url": build_url(ch)}) + orid = (model.get("orchestrator") or {}).get("run_id") + if orid: + out.append({"name": "Release Orchestrator run", "url": build_url(orid)}) + for prov in ("ECS", "Local"): + rid = ((model.get("mrwp") or {}).get(prov) or {}).get("run_id") + if rid: + out.append({"name": f"MRWP {prov} run", "url": build_url(rid)}) + return out + + +def rc_ui_gate(model) -> dict: + """The Phase-2 RC quality gate. Aggregates UI-automation results across BOTH MRWP + providers (ECS + Local) and decides whether RC quality clears the bar. Returns + {ui_total, ui_passed, ui_failed, pass_pct, threshold, verdict, detail} + `verdict` is 'pass' when pass_pct >= RC_UI_PASS_THRESHOLD (or no UI tests were found), + else 'attention' (below the bar → the rc_report step must block for investigation). + `detail` is the human note recorded on the step / shown to the owner.""" + ui_total = ui_pass = ui_fail = 0 + for prov in ("ECS", "Local"): + ui = (((model.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) \ + .get("categories", {}).get("ui") or {} + ui_total += ui.get("total") or 0 + ui_pass += ui.get("passed") or 0 + ui_fail += ui.get("failed") or 0 + thr = RC_UI_PASS_THRESHOLD + if not ui_total: + return {"ui_total": 0, "ui_passed": 0, "ui_failed": 0, "pass_pct": None, + "threshold": thr, "verdict": "pass", + "detail": ("\u26a0 No UI-automation tests were found in either MRWP run — " + "nothing to gate on. Proceeding, but verify RC test coverage.")} + pass_pct = round(ui_pass * 100.0 / ui_total, 1) + ok = pass_pct >= thr + head = (f"UI-automation pass rate {pass_pct}% ({ui_pass}/{ui_total} passed, " + f"{ui_fail} failed) across ECS + Local") + if ok: + detail = f"{head} \u2014 at or above the {thr:.0f}% gate. RC quality OK to proceed." + else: + detail = (f"{head} \u2014 BELOW the {thr:.0f}% gate. This is a large UI failure; " + f"investigate the root cause (it likely needs a fix + an MRWP re-run). " + f"The failing suites are in the RC report email. Once fixed and re-run, " + f"re-run this step (`next`); to override, `skip`.") + suites = [] + for prov in ("ECS", "Local"): + for s in (((model.get("mrwp") or {}).get(prov) or {}).get("failed_suites") or []): + if s.get("category", "ui") == "ui" and s.get("failed"): + suites.append((prov, s)) + suites.sort(key=lambda ps: -ps[1]["failed"]) + if suites: + detail += "\nTop UI failures:\n" + "\n".join( + f" \u2022 [{prov}] {s['name']}: {s['failed']}/{s['total']} failed" + for prov, s in suites[:6]) + return {"ui_total": ui_total, "ui_passed": ui_pass, "ui_failed": ui_fail, + "pass_pct": pass_pct, "threshold": thr, + "verdict": "pass" if ok else "attention", "detail": detail} + + def rc_email_subject(model) -> str: rid = model.get("release", "?") - return (f"Release {rid} — RC verification report (Phase 2) · " - f"action: approve to proceed to bug bash") + action = ("approve to proceed to bug bash" + if rc_ui_gate(model)["verdict"] == "pass" + else "investigate UI failures before proceeding") + return f"Release {rid} — RC verification report (Phase 2) · action: {action}" def _fail_rate(failed, total) -> float: diff --git a/release-agent/steps/build_verify/rc_report.py b/release-agent/steps/build_verify/rc_report.py index a8b337e0..6a94851c 100644 --- a/release-agent/steps/build_verify/rc_report.py +++ b/release-agent/steps/build_verify/rc_report.py @@ -1,15 +1,23 @@ -"""Step: `rc_report` — email the RC verification report to the release owner -(Phase 2, build_verify), right before the go_test approval gate. +"""Step: `rc_report` — email the RC verification report to the release owner AND apply +the Phase-2 UI-automation quality gate (Phase 2, build_verify), right before the +go_test approval gate. When the four verification steps have resolved the chain, this step composes the Phase-2 RC report (checker → orchestrator → ECS/Local MRWP + per-run test failures) from LIVE pipeline data and emails it to the release owner, so the engineer wakes to -the report on CCD+1 and can review before approving `go_test`. +the report on CCD+1. The report is ALWAYS sent (the owner gets the dashboard of +failures + links either way). The step's OUTCOME is then decided by the UI gate: if the +UI-automation pass rate across both MRWP runs is >= RC_UI_PASS_THRESHOLD (90%) it +records `pass` and the flow advances to go_test; below the bar it records `attention` +(the step BLOCKS) so the owner investigates the large failure (usually a fix + MRWP +re-run) before proceeding. Sending email needs the WorkIQ MCP the engine can't reach, so this is a `scout` step: `build()` composes the email deterministically and returns a -NeedsSkill(workiq_send_email) for the skill to send. Redirect for tests with the -`send_to` payload knob (keeps the send real, points it at you). +NeedsSkill(workiq_send_email); the payload names the `record-rc-report` follow-up +command, which re-reads the model, applies the gate, records pass|attention, and +stashes the evaluated run links on the step. Redirect for tests with the `send_to` +payload knob (keeps the send real, points it at you). """ from __future__ import annotations @@ -29,7 +37,9 @@ def build(state): """Compose the RC verification email → NeedsSkill(workiq_send_email). Blocks if the - owner email is unknown (nowhere to send) — set it with `set-owner`.""" + owner email is unknown (nowhere to send) — set it with `set-owner`. The email is + always sent; the UI gate verdict (recorded by the `record-rc-report` follow-up) then + decides whether the step passes or blocks.""" to = state.owner_email if not to: return Blocked( @@ -40,8 +50,14 @@ def build(state): except Exception as e: # pragma: no cover - defensive return Blocked(f"rc_report: could not build the RC report ({e}).") - probs = model.get("problems") or [] - tail = f"; {len(probs)} blocking issue(s)" if probs else "" + gate = K.rc_ui_gate(model) + if gate["verdict"] == "pass": + summary = (f"Email the RC verification report to the release owner ({to}) — " + f"UI gate PASS") + else: + summary = (f"Email the RC verification report to the release owner ({to}) — " + f"UI gate FAIL ({gate['pass_pct']}% < {int(K.RC_UI_PASS_THRESHOLD)}%); " + f"will block for investigation") return NeedsSkill( tool="workiq_send_email", payload={ @@ -50,9 +66,12 @@ def build(state): "body": html, "isHtml": True, "_plain_body": plain, + # After sending, DON'T blind-record pass: run this engine command instead — + # it applies the 90% UI gate (pass|attention) and stashes the run links. + "followup_command": "record-rc-report", }, record_as=ID, - summary=f"Email the RC verification report to the release owner ({to}){tail}", - note=f"RC report emailed to {to}", + summary=summary, + note=gate["detail"], outbound=True, ) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 8f108857..6624a055 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2039,6 +2039,119 @@ def test_build_verify_rc_report_emails_owner(): P.release_report = orig +def test_rc_ui_gate_and_run_links(): + """The Phase-2 UI gate aggregates UI-automation results across BOTH MRWP providers: + >=90% combined pass → 'pass'; below → 'attention' (with a failing-suite summary); + no UI tests → pass with a warning. rc_run_links surfaces every evaluated run.""" + from steps.build_verify import _common as K + + def _model(ecs_ui, local_ui, ecs_suites=None): + return { + "release": "2026-08", + "checker": {"fired": True, "run_id": 111}, + "orchestrator": {"found": True, "run_id": 222}, + "mrwp": { + "ECS": {"run_id": 333, "failed_suites": ecs_suites or [], + "tests": {"categories": {"ui": ecs_ui}}}, + "Local": {"run_id": 444, "failed_suites": [], + "tests": {"categories": {"ui": local_ui}}}}} + + # 180/200 = 90.0% → exactly at the bar → pass + g = K.rc_ui_gate(_model({"total": 100, "passed": 100, "failed": 0}, + {"total": 100, "passed": 80, "failed": 20})) + assert g["verdict"] == "pass" and g["pass_pct"] == 90.0 and g["ui_total"] == 200 + + # 160/200 = 80% → below the bar → attention, with the failing suite listed + fail_model = _model({"total": 100, "passed": 60, "failed": 40}, + {"total": 100, "passed": 100, "failed": 0}, + ecs_suites=[{"name": "PROD MSAL - RC Broker (API 32)", + "failed": 40, "total": 100, "category": "ui"}]) + g2 = K.rc_ui_gate(fail_model) + assert g2["verdict"] == "attention" and g2["pass_pct"] == 80.0 + assert "BELOW" in g2["detail"] and "PROD MSAL - RC Broker (API 32)" in g2["detail"] + + # no UI tests anywhere → pass with a warning (absence of data is not a failure) + g3 = K.rc_ui_gate({"mrwp": {"ECS": {"tests": {"categories": {}}}, + "Local": {"tests": {"categories": {}}}}}) + assert g3["verdict"] == "pass" and g3["ui_total"] == 0 and "No UI-automation" in g3["detail"] + + # every evaluated run becomes a durable link + links = K.rc_run_links(fail_model) + names = [l["name"] for l in links] + assert names == ["Code Complete Checker run", "Release Orchestrator run", + "MRWP ECS run", "MRWP Local run"] + assert all("buildId=" in l["url"] for l in links) + + +def test_record_rc_report_applies_ui_gate_and_stashes_links(): + """`record-rc-report` (the follow-up the skill runs after emailing) applies the 90% + UI gate: >=90% → step done; <90% → step BLOCKS (awaiting_action). Either way it + stashes the evaluated run links on the step. release_report is monkeypatched offline.""" + import tempfile as _tf + from tools import pipelines as P + from orchestrator.commands import rc_report as RR + from orchestrator.state import StepState + + def _model(ecs_ui, local_ui): + return {"release": "2026-08", + "checker": {"fired": True, "run_id": 111}, + "orchestrator": {"found": True, "run_id": 222}, + "mrwp": {"ECS": {"run_id": 333, "failed_suites": [], + "tests": {"categories": {"ui": ecs_ui}}}, + "Local": {"run_id": 444, "failed_suites": [], + "tests": {"categories": {"ui": local_ui}}}}, + "problems": []} + + orig = P.release_report + with _tf.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + st = ReleaseState(release_id=rid, ccd="2026-08-26", owner_email="dev@microsoft.com") + orch = Orchestrator(CONFIG, st) + _pass_scout_checks(orch); orch.gate.sign() + C.save_state(st, d, rid) + + class A: + runs_root = d; release = rid; config = CONFIG; as_of = None + + try: + # PASS: 190/200 = 95% ≥ 90 → step done, links stashed + P.release_report = lambda *a, **k: _model( + {"total": 100, "passed": 95, "failed": 5}, + {"total": 100, "passed": 95, "failed": 5}) + assert RR.cmd_record_rc_report(A) == 0 + s1 = C.load_state(d, rid) + assert s1.is_done("build_verify", "rc_report") + step1 = s1.get_step("build_verify", "rc_report") + assert [l["name"] for l in step1.links] == [ + "Code Complete Checker run", "Release Orchestrator run", + "MRWP ECS run", "MRWP Local run"] + + # reset the step, then FAIL: 120/200 = 60% < 90 → blocked, links still stashed + s1.set_step("build_verify", "rc_report", StepState()) + C.save_state(s1, d, rid) + P.release_report = lambda *a, **k: _model( + {"total": 100, "passed": 60, "failed": 40}, + {"total": 100, "passed": 60, "failed": 40}) + assert RR.cmd_record_rc_report(A) == 2 + s2 = C.load_state(d, rid) + step2 = s2.get_step("build_verify", "rc_report") + assert step2.status == "blocked" and not s2.is_done("build_verify", "rc_report") + assert s2.status == "awaiting_action" + assert "build_verify.rc_report" in s2.pending_human + assert "BELOW" in step2.note and len(step2.links) == 4 + finally: + P.release_report = orig + + +def test_active_phase_report_steps_carry_links(): + """The digest's active-phase step model exposes each step's durable `links` so the + links to items a step evaluated aren't dropped from the phase report.""" + st, orch = _mock_orch({}) + ap = orch.status_report()["active_phase"] + assert ap and all("links" in s for s in ap["steps"]) + + def test_classify_test_run_categories(): """The test-run classifier buckets into exactly three: unit / instrumented / ui; anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" From 4da6e02ff4e7502702cb4aec60a30ee6e4577f24 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 12:12:18 -0700 Subject: [PATCH 65/90] Refactor Phase 2 RC verification process: remove go_test gate, implement three-tier UI automation quality gate - Updated build_verify_live.yaml to auto-advance rc_report and position at bug-bash entry. - Modified mocks.local.example.yaml to reflect changes in RC report handling. - Enhanced rc_report.py to apply a three-tier UI gate, determining pass/warn/attention outcomes. - Adjusted sim.py to change auto-approve gates from go_test to bash_done. - Revised SKILL.md and commands.md to update user guidance on RC verification and command usage. - Updated build_verify.md to clarify the removal of the go_test gate and the new RC report process. - Enhanced _common.py to summarize UI failing suites for better reporting. - Updated tests to reflect the removal of the go_test gate and ensure correct behavior in the new flow. --- release-agent/config/knowledge.yaml | 22 ++- release-agent/config/phases.yaml | 6 +- .../config/scenarios/at_rc_gate.yaml | 12 +- .../config/scenarios/build_verify_live.yaml | 7 +- release-agent/mocks.local.example.yaml | 2 +- .../orchestrator/commands/rc_report.py | 11 +- release-agent/orchestrator/sim.py | 2 +- release-agent/skill/SKILL.md | 4 +- release-agent/skill/reference/commands.md | 2 +- .../skill/reference/phases/build_verify.md | 39 +++-- release-agent/steps/build_verify/_common.py | 82 +++++---- release-agent/steps/build_verify/rc_report.py | 25 ++- release-agent/tests/test_engine.py | 160 ++++++++++-------- 13 files changed, 216 insertions(+), 158 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index d83fdf5c..7aceb7db 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -309,26 +309,30 @@ build_verify.mrwp_local: a: "The orchestrator triggers MRWP twice with different flight providers — ECS (server-driven flights) and Local (local flight overrides). Both must run to completion; they're verified separately." build_verify.rc_report: - summary: "Email the RC verification report (pipeline health + test failures) to the release owner." + summary: "Email the RC verification report + apply the three-tier 90% UI gate (the Phase-2 go/no-go)." what: > Once the four verification steps have resolved the chain, this composes the Phase-2 RC report — checker fired, orchestrator healthy/parked, both MRWP runs' stage completion, and each run's - failing test suites — from LIVE pipeline data and emails it to the release owner. It runs right - before the go_test gate so the engineer wakes (on CCD+1) to the report and can review before - approving. Scout-assisted (source: scout): the engine composes it; the skill sends via WorkIQ. + failing test suites — from LIVE pipeline data and emails it to the release owner (on CCD+1). It + is the terminal Phase-2 step and the go/no-go: after sending, `record-rc-report` applies a + three-tier gate on the combined UI-automation pass rate across both MRWP runs — 100% is a clean + pass, >=90% passes with a warning (investigate the failing tests in parallel, bug bash not + blocked), and <90% BLOCKS for owner investigation. There is no separate approval gate. + Scout-assisted (source: scout): the engine composes it; the skill sends via WorkIQ. who: > Scout composes the email; the skill sends it via workiq_send_email to the release owner (state.owner_email). Redirect for testing with the step's `send_to` payload knob. where: - - "Release Orchestrator run (versions + parked gate): via the run link in the email" - - "MRWP ECS / Local runs (Test tab): via the run links in the email" + - "Release Orchestrator run (versions + parked gate): via the run link in the email + the step's Details" + - "MRWP ECS / Local runs (Test tab): via the run links in the email + the step's Details" how: > If it blocks with 'no release owner email', set it with `set-owner --email ` - and rerun. The email frames failing tests as bug-bash triage (not blockers) unless a stage never - ran. After it sends, the go_test gate holds for the owner's approval. + and rerun. On a clean/warn pass (>=90% UI) the release auto-advances into bug bash. On a <90% + block, the owner investigates and rules on it (patch a real bug + re-trigger RC, or proceed as an + automation flake): fix + re-run then `next`, or `skip --reason` to override. faqs: - q: "Who gets the email?" - a: "The release owner on record (owner_email). It's a personal review copy, not the broad DL — the owner reviews the RC failures, then approves the go_test gate." + a: "The release owner on record (owner_email). It's a personal review copy, not the broad DL — the owner reviews the RC failures; the 90% UI gate then decides go/no-go automatically." - q: "Does the sim send this email?" a: "No. In a sim the step is marked done without sending; only the real skill flow composes + sends it via WorkIQ." diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 982936c5..e07fcda3 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -53,8 +53,10 @@ phases: - { id: orchestrator_health, name: "Verify Release Orchestrator health (parked at Remove RC Tags)", owner: agent, maps_to: [B1] } - { id: mrwp_ecs, name: "Verify MRWP (ECS) ran to completion + tests", owner: agent, maps_to: [B2] } - { id: mrwp_local, name: "Verify MRWP (Local) ran to completion + tests", owner: agent, maps_to: [B3] } - - { id: rc_report, name: "Email RC verification report to release owner", owner: agent, source: scout, maps_to: [B4] } - - { id: go_test, name: "RC verified — proceed to bug bash", owner: human, gate: true } + # rc_report is the Phase-2 go/no-go: it emails the RC report AND applies the 90% UI + # gate — clean/warn (>=90%) proceed to bug bash; <90% BLOCKS for owner investigation. + # No separate human approval step: the gate IS the decision. + - { id: rc_report, name: "Email RC verification report + apply the 90% UI gate", owner: agent, source: scout, maps_to: [B4] } - id: bug_bash name: "Test / Bug Bash" diff --git a/release-agent/config/scenarios/at_rc_gate.yaml b/release-agent/config/scenarios/at_rc_gate.yaml index 084f24ac..34585316 100644 --- a/release-agent/config/scenarios/at_rc_gate.yaml +++ b/release-agent/config/scenarios/at_rc_gate.yaml @@ -1,7 +1,9 @@ -# Fast-forward all the way to Phase 2's go_test gate — the four verification steps -# resolve from deterministic OFFLINE mocks (no az), leaving the release holding at the -# human "RC verified — proceed to bug bash" decision. Use this to exercise the gate -# hold, the status render, and the RC digest one-liner without touching the network. +# Fast-forward THROUGH Phase 2 (Build & RC Verification): the four verification steps +# resolve from deterministic OFFLINE mocks (no az) and stash the pipeline ids, then the +# rc_report step auto-advances (go_test was removed — rc_report's 90% UI gate is the +# decision). Lands positioned at the entry of Phase 3 (bug bash). Use this to exercise +# the RC verification chain, the pipeline-id stash, and the status render without touching +# the network. # # release-agent sim run --scenario at_rc_gate # @@ -10,7 +12,7 @@ release_id: 2026-08 ccd: 2026-08-26 ccd_source: confirmed as_of: CCD+2 -target: { phase: build_verify, at: gate } +target: { phase: build_verify, at: done } data: mock mocks: build_verify.checker_fired: diff --git a/release-agent/config/scenarios/build_verify_live.yaml b/release-agent/config/scenarios/build_verify_live.yaml index 0c214fcb..6d8b83f8 100644 --- a/release-agent/config/scenarios/build_verify_live.yaml +++ b/release-agent/config/scenarios/build_verify_live.yaml @@ -1,7 +1,8 @@ # Test Phase 2 END-TO-END against the REAL pipelines: fast-forwards Phases 0-1, # then runs the four verification steps LIVE (real `az` reads of the 2026-08 -# checker → orchestrator → ECS/Local MRWP runs) and lands holding at the go_test -# gate. Seeds the real release, so afterwards use the normal skill: status / approve. +# checker → orchestrator → ECS/Local MRWP runs), auto-advances rc_report, and lands +# positioned at the bug-bash entry (go_test was removed — rc_report's 90% UI gate is the +# decision). Seeds the real release, so afterwards use the normal skill: status / next. # # release-agent sim run --scenario build_verify_live # @@ -10,5 +11,5 @@ release_id: 2026-08 ccd: 2026-08-26 ccd_source: confirmed as_of: CCD+2 -target: { phase: build_verify, at: gate } +target: { phase: build_verify, at: done } data: live # the 4 build_verify steps hit real az; earlier phases are mocked diff --git a/release-agent/mocks.local.example.yaml b/release-agent/mocks.local.example.yaml index ce6e0710..7929d6b9 100644 --- a/release-agent/mocks.local.example.yaml +++ b/release-agent/mocks.local.example.yaml @@ -81,7 +81,7 @@ preflight.cg: # build_verify.ui_auto: { outcome: done } # agent # build_verify.payload: { outcome: done } # agent # build_verify.mrwp_rc: { outcome: done } # agent -# build_verify.go_test: 🚦 gate — NOT mockable +# build_verify.rc_report: { outcome: done } # scout (emails RC report + applies the 90% UI gate) # ---- Phase 3 · bug_bash ---- # bug_bash.clone_plans: { outcome: done } # agent diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 08473d64..d641021d 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -55,10 +55,11 @@ def _u(build_id): def cmd_record_rc_report(args): """Record the rc_report step's outcome AFTER the skill has emailed the RC report. - Re-reads the live model, applies the UI-automation quality gate (K.rc_ui_gate), - records `pass` (>=90% UI pass → step done, flow advances to go_test) or `attention` - (<90% → step BLOCKS for owner investigation), and stashes the evaluated pipeline-run - links on the step so its Details point at every artifact behind the verdict. + Re-reads the live model, applies the three-tier UI-automation gate (K.rc_ui_gate), + records `pass` (>=90% UI pass — clean/warn → step done, release auto-advances into bug + bash) or `attention` (<90% → step BLOCKS for owner investigation), and stashes the + evaluated pipeline-run links on the step so its Details point at every artifact behind + the verdict. This is the follow-up the rc_report NeedsSkill names (`payload.followup_command`), so the skill runs it instead of a blind `record-step --status pass`.""" @@ -71,7 +72,7 @@ def cmd_record_rc_report(args): gate = K.rc_ui_gate(model) links = K.rc_run_links(model) - status = "pass" if gate["verdict"] == "pass" else "attention" + status = "attention" if gate["blocking"] else "pass" orch.record_scout_step("build_verify", "rc_report", status, gate["detail"]) # record_scout_step doesn't carry links — attach the evaluated-run refs (and stamp diff --git a/release-agent/orchestrator/sim.py b/release-agent/orchestrator/sim.py index b188d1c3..79304ed0 100644 --- a/release-agent/orchestrator/sim.py +++ b/release-agent/orchestrator/sim.py @@ -24,7 +24,7 @@ as_of: CCD+1 # CCD-relative or absolute YYYY-MM-DD; default: ccd target: {phase: build_verify, at: open} # at: open | gate | done data: live # live | mock (default: mock) - approve_gates: [go_test] # gates to auto-approve while fast-forwarding (see note) + approve_gates: [bash_done] # gates to auto-approve while fast-forwarding (see note) mocks: # merged last (win); fine-grained inputs or outcome mocks build_verify.checker_fired: { triggering: {...} } seed: # direct ReleaseState field overrides (e.g. pipeline_runs) diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index 56bb23f9..b0f207a4 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -43,7 +43,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → - **User asks about the RC pipelines / RC tests / "phase 2 status"** ("how are the release pipelines?", "did the RC tests pass?", "show pipeline status", "is the orchestrator done?"): run **`rc-report --release `** and paste its output verbatim — the checker → orchestrator → ECS/Local-MRWP chain with each run's stage completion + Test-tab breakdown (unit/instrumented/UI-automation). It's **read-only** (never gates); use `--json` for your own branching. Red/yellow stages and failed tests are expected here (triaged in bug bash) — only a stage that never ran is a real problem, and it shows under **Issues**. - **User wants to test/validate a phase mid-release without running one from scratch** ("test phase 2", "simulate phase 2", "let me test the RC phase", "drop me in at the RC gate", "test the bug-bash phase"): this is the **sim** — it SEEDS the real release to a mid-release point so you then drive it with the normal skill. Run it for them via the shell; don't hand them python. Pick the scenario by intent (`sim list` shows all): - - "test phase 2" / "test phase 2 against the real pipelines" / "does phase 2 work" → **`sim run --scenario build_verify_live`** (fast-forwards Phases 0-1, runs the 4 verification steps against the **real** 2026-08 `az` runs, lands holding at the go_test gate). + - "test phase 2" / "test phase 2 against the real pipelines" / "does phase 2 work" → **`sim run --scenario build_verify_live`** (fast-forwards Phases 0-1, runs the 4 verification steps against the **real** 2026-08 `az` runs, auto-advances rc_report, lands positioned at the bug-bash entry). - "test phase 2 offline / quickly / without the network" → **`sim run --scenario at_rc_gate`** (same flow, fully mocked). - "drop me at phase 2 so I can step through it myself" → **`sim run --scenario mid_build_verify_open`** (positions at entry, runs nothing — then use `next` to run each step live). @@ -55,7 +55,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → | Running the readiness entry gate (right after `init`) | `reference/readiness-gate.md` | | Starting a release / handling CCD / setting up push reminders & automations | `reference/starting-and-scheduling.md` | | Advancing **Phase 0 (Pre-flight)** — notice, flight reminders, lockdown, confirm, vitals | `reference/phases/preflight.md` | -| Advancing **Phase 2 (Build & RC Verification)** — verification chain, RC report email + 90% UI gate, go_test | `reference/phases/build_verify.md` | +| Advancing **Phase 2 (Build & RC Verification)** — verification chain, RC report email + three-tier 90% UI gate (no separate gate) | `reference/phases/build_verify.md` | | Rendering `status`/`checklist` output | `reference/presenting-status.md` | | Looking up a command / manual override / event-logging detail | `reference/commands.md` | | Building a NEW phase's guidance | `reference/phases/_TEMPLATE.md` | diff --git a/release-agent/skill/reference/commands.md b/release-agent/skill/reference/commands.md index 96110fb1..f7e87faf 100644 --- a/release-agent/skill/reference/commands.md +++ b/release-agent/skill/reference/commands.md @@ -15,7 +15,7 @@ _Loaded on demand. Run all from `C:\repos\android-complete\release-agent`._ | Resolve a migrated step → outcome JSON (done\|blocked\|needs_human\|needs_skill) | `python -m orchestrator.cli step-action --release --step [--phase

    ] [--param k=v …]` | | Answer a STEP question (knowledge) | `python -m orchestrator.cli step-info --step [--phase

    ]` | | **Phase 2 — RC pipeline + test report** (read-only) | `python -m orchestrator.cli rc-report --release [--json]` → the checker→orchestrator→ECS/Local-MRWP chain + per-run test breakdown | -| **Phase 2 — record RC verdict** (after emailing the report) | `python -m orchestrator.cli record-rc-report --release ` → applies the **90% UI-automation gate** across both MRWP runs, records `pass` (step done → go_test) or `attention` (step **blocks** for investigation), and stashes the checker/orchestrator/ECS/Local run links on the step. This is the follow-up the `rc_report` `needs_skill` names — run it **instead of** `record-step` | +| **Phase 2 — record RC verdict** (after emailing the report) | `python -m orchestrator.cli record-rc-report --release ` → applies the **three-tier 90% UI-automation gate** across both MRWP runs, records `pass` (100% clean / ≥90% warn → step done, release auto-advances into bug bash) or `attention` (<90% → step **blocks** for investigation), and stashes the checker/orchestrator/ECS/Local run links on the step. This is the follow-up the `rc_report` `needs_skill` names — run it **instead of** `record-step` | | **Simulate a mid-release point** (testing) | `python -m orchestrator.cli sim list` · `python -m orchestrator.cli sim run --scenario [--freeze] [--json]` → **seeds the real release** to a scenario's target (`config/scenarios/.yaml`): fast-forwards the real engine, signs the entry gate + completes earlier phases from mocks, then stops `open`/`gate`/`done` at the target. `data: live` runs the target phase against real `az`; `data: mock` is offline. Any existing state at that id is backed up first, so afterwards you use the **normal** commands (`status`, `rc-report`, `next`, `approve`). `--runs-root ` targets a throwaway sandbox instead; `--freeze` snapshots state to `tests/fixtures/.json` | | Answer an ENTRY-GATE item question (knowledge) | `python -m orchestrator.cli gate-info --item ` (build_access, mcp_servers, ccd_confirmed, silent_perms, teams_notify, adx_access, oncall_now, play_console_access, oncall_window, saw_ame, yubikey) | | Prepare early code-complete notice (JSON) — _legacy; prefer `step-action --step notice`_ | `python -m orchestrator.cli prepare-notice --release [--variant initial\|update]` | diff --git a/release-agent/skill/reference/phases/build_verify.md b/release-agent/skill/reference/phases/build_verify.md index e6163f38..79e4b22e 100644 --- a/release-agent/skill/reference/phases/build_verify.md +++ b/release-agent/skill/reference/phases/build_verify.md @@ -2,21 +2,23 @@ Opens **CCD+1** — the engineer wakes to a resume. The engine runs the four verification **agent** steps in-process during `next`; you relay their results and drive the one -**scout** step (`rc_report`) + the human **gate** (`go_test`). +**scout** step (`rc_report`), which is the terminal Phase-2 step **and** the go/no-go — +there is no separate human gate. ## Execution model Sequential. A single `next` runs the agent chain (checker → orchestrator → ECS/Local MRWP); each **blocks** on a real problem (a stage that never ran, an unhealthy orchestrator, an auth failure). A blocked step → show the note, then **fix + `next`** to re-check, or **`skip … --reason`** to override. When the chain is green the scout -`rc_report` step becomes ready, then the `go_test` gate. +`rc_report` step runs — it is the last Phase-2 step and the go/no-go. ## Automated steps (no skill action — relay from the `status` table) `checker_fired`, `orchestrator_health`, `mrwp_ecs`, `mrwp_local` — read-only `az` agent steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 link. `step-action` refuses them (exit 1); never dispatch them yourself. -## `rc_report` — email the RC report + apply the 90% UI gate (`scout`) +## `rc_report` — email the RC report + apply the 90% UI gate (`scout`, terminal) +This is the Phase-2 go/no-go — there is **no separate approval gate**. - **Trigger:** `status --json` shows current step `rc_report` (state `scout`), after the four agent steps are done. - **Resolve:** `step-action --release --phase build_verify --step rc_report` → @@ -24,23 +26,24 @@ steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 `payload.followup_command: record-rc-report`. - **Act:** send the email verbatim (`payload.to/subject/body`, `isHtml:true`) — honoring a `send_to` redirect if the engineer set one. **Always send** — the owner gets the - dashboard (failing suites + run links) whether the gate passes or not. + dashboard (failing suites + run links) whatever the verdict is. - **Record (the two-hop):** because `followup_command` is set, run `record-rc-report --release ` **instead of** `record-step`. It re-reads the model, - applies the **90% UI-automation gate** (combined pass rate across ECS + Local): - - **≥ 90% → `pass`** — the step is done; advance to `go_test`. - - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). This is a large UI - failure: the owner must **investigate the root cause** (usually a **fix + an MRWP - re-run**). Exits: fix + re-run, then `next` re-runs `rc_report`; or `skip … --reason` - to override. It records the failing-suite summary + stashes the checker/orchestrator/ - ECS/Local run links on the step. - - (No UI tests found → passes with a ⚠ note.) -- The command prints `{verdict, pass_pct, ui_total, detail, links}` for your branching; - relay the `status` table (the `rc_report` Details shows the verdict + 🔗 links). - -## `go_test` — RC verified, proceed to bug bash (`gate`, human) -Present the settled `status`; `m_ask_user` Approve/Deny; `approve` / `deny --comment`. -Never authorize yourself. If `rc_report` blocked, `go_test` isn't reached until it clears. + applies the **three-tier 90% UI-automation gate** (combined pass rate across ECS + Local): + - **100% → `clean`** — step done; the release auto-advances into Phase 3 (bug bash). + - **≥ 90% & < 100% → `warn`** — step done; auto-advances into bug bash, but the owner + should investigate the failing UI tests **in parallel** (a later step confirms the + retest — bug bash is **not** blocked). + - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). Large UI failure: the + owner investigates and decides — patch a real bug + re-trigger RC, or (if it's an + automation flake to re-run later) proceed to bug bash. Exits: fix + re-run, then + `next` re-runs `rc_report`; or `skip … --reason` to override. + - (No UI tests found → `clean` with a ⚠ note.) + It records the failing-suite summary + stashes the checker/orchestrator/ECS/Local run + links on the step. +- The command prints `{verdict, blocking, pass_pct, ui_total, detail, links}` for your + branching; relay the `status` table (the `rc_report` Details shows the verdict + 🔗 links). + On `clean`/`warn` the engine auto-advances into Phase 3 — brief the owner and continue. ## External references Engineering pipelines: Checker def 3038, Orchestrator def 2828, MRWP def 2519 diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 7d199c98..629adf50 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -93,13 +93,35 @@ def rc_run_links(model) -> list: return out +def _ui_failing_suites_summary(model, limit=6) -> str: + """A compact 'Top UI failures' list across both providers, or '' when none.""" + suites = [] + for prov in ("ECS", "Local"): + for s in (((model.get("mrwp") or {}).get(prov) or {}).get("failed_suites") or []): + if s.get("category", "ui") == "ui" and s.get("failed"): + suites.append((prov, s)) + suites.sort(key=lambda ps: -ps[1]["failed"]) + if not suites: + return "" + return "\nTop UI failures:\n" + "\n".join( + f" \u2022 [{prov}] {s['name']}: {s['failed']}/{s['total']} failed" + for prov, s in suites[:limit]) + + def rc_ui_gate(model) -> dict: - """The Phase-2 RC quality gate. Aggregates UI-automation results across BOTH MRWP - providers (ECS + Local) and decides whether RC quality clears the bar. Returns - {ui_total, ui_passed, ui_failed, pass_pct, threshold, verdict, detail} - `verdict` is 'pass' when pass_pct >= RC_UI_PASS_THRESHOLD (or no UI tests were found), - else 'attention' (below the bar → the rc_report step must block for investigation). - `detail` is the human note recorded on the step / shown to the owner.""" + """The Phase-2 RC quality gate — a THREE-tier decision on the combined UI-automation + pass rate across both MRWP providers (ECS + Local). Returns + {ui_total, ui_passed, ui_failed, pass_pct, threshold, verdict, blocking, detail} + where `verdict` is: + * 'clean' — 100% UI pass (or no UI tests found): proceed, no action. + * 'warn' — >= RC_UI_PASS_THRESHOLD (90%) but < 100%: proceed to bug bash, but + the owner should investigate the failing UI tests IN PARALLEL (a + later step confirms the retest — bug bash is NOT blocked). + * 'attention' — < 90%: BLOCK. A large failure the owner must investigate and rule + on (patch a real bug + re-trigger RC, or proceed as an automation + flake to re-run later). + `blocking` is True only for 'attention'. `detail` is the note recorded on the step / + shown to the owner.""" ui_total = ui_pass = ui_fail = 0 for prov in ("ECS", "Local"): ui = (((model.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) \ @@ -108,42 +130,38 @@ def rc_ui_gate(model) -> dict: ui_pass += ui.get("passed") or 0 ui_fail += ui.get("failed") or 0 thr = RC_UI_PASS_THRESHOLD + base = {"ui_total": ui_total, "ui_passed": ui_pass, "ui_failed": ui_fail, "threshold": thr} if not ui_total: - return {"ui_total": 0, "ui_passed": 0, "ui_failed": 0, "pass_pct": None, - "threshold": thr, "verdict": "pass", + return {**base, "pass_pct": None, "verdict": "clean", "blocking": False, "detail": ("\u26a0 No UI-automation tests were found in either MRWP run — " "nothing to gate on. Proceeding, but verify RC test coverage.")} pass_pct = round(ui_pass * 100.0 / ui_total, 1) - ok = pass_pct >= thr head = (f"UI-automation pass rate {pass_pct}% ({ui_pass}/{ui_total} passed, " f"{ui_fail} failed) across ECS + Local") - if ok: - detail = f"{head} \u2014 at or above the {thr:.0f}% gate. RC quality OK to proceed." - else: - detail = (f"{head} \u2014 BELOW the {thr:.0f}% gate. This is a large UI failure; " - f"investigate the root cause (it likely needs a fix + an MRWP re-run). " - f"The failing suites are in the RC report email. Once fixed and re-run, " - f"re-run this step (`next`); to override, `skip`.") - suites = [] - for prov in ("ECS", "Local"): - for s in (((model.get("mrwp") or {}).get(prov) or {}).get("failed_suites") or []): - if s.get("category", "ui") == "ui" and s.get("failed"): - suites.append((prov, s)) - suites.sort(key=lambda ps: -ps[1]["failed"]) - if suites: - detail += "\nTop UI failures:\n" + "\n".join( - f" \u2022 [{prov}] {s['name']}: {s['failed']}/{s['total']} failed" - for prov, s in suites[:6]) - return {"ui_total": ui_total, "ui_passed": ui_pass, "ui_failed": ui_fail, - "pass_pct": pass_pct, "threshold": thr, - "verdict": "pass" if ok else "attention", "detail": detail} + if pass_pct >= 100.0: + return {**base, "pass_pct": pass_pct, "verdict": "clean", "blocking": False, + "detail": (f"UI-automation pass rate 100% ({ui_pass}/{ui_total}) — all UI " + f"tests passed. Proceeding to bug bash.")} + if pass_pct >= thr: + return {**base, "pass_pct": pass_pct, "verdict": "warn", "blocking": False, + "detail": (f"{head} \u2014 at or above the {thr:.0f}% gate but not clean. " + f"Proceeding to bug bash; release owner: investigate the {ui_fail} " + f"failing UI test(s) in parallel (a later step confirms the retest, " + f"so bug bash is not blocked)." + _ui_failing_suites_summary(model))} + return {**base, "pass_pct": pass_pct, "verdict": "attention", "blocking": True, + "detail": (f"{head} \u2014 BELOW the {thr:.0f}% gate. Large UI failure: investigate " + f"the root cause and decide \u2014 patch a real bug + re-trigger RC, or " + f"(if it's an automation flake to re-run later) proceed to bug bash. This " + f"step stays BLOCKED until you `next` after a re-run, or `skip --reason` " + f"to override." + _ui_failing_suites_summary(model))} def rc_email_subject(model) -> str: rid = model.get("release", "?") - action = ("approve to proceed to bug bash" - if rc_ui_gate(model)["verdict"] == "pass" - else "investigate UI failures before proceeding") + v = rc_ui_gate(model)["verdict"] + action = {"clean": "approve to proceed to bug bash", + "warn": "proceeding to bug bash — investigate failing UI tests in parallel", + "attention": "investigate UI failures before proceeding"}[v] return f"Release {rid} — RC verification report (Phase 2) · action: {action}" diff --git a/release-agent/steps/build_verify/rc_report.py b/release-agent/steps/build_verify/rc_report.py index 6a94851c..87252ee9 100644 --- a/release-agent/steps/build_verify/rc_report.py +++ b/release-agent/steps/build_verify/rc_report.py @@ -1,16 +1,18 @@ """Step: `rc_report` — email the RC verification report to the release owner AND apply -the Phase-2 UI-automation quality gate (Phase 2, build_verify), right before the -go_test approval gate. +the Phase-2 UI-automation quality gate (Phase 2, build_verify). This is the terminal +Phase-2 step and the go/no-go — there is no separate human approval gate. When the four verification steps have resolved the chain, this step composes the Phase-2 RC report (checker → orchestrator → ECS/Local MRWP + per-run test failures) from LIVE pipeline data and emails it to the release owner, so the engineer wakes to the report on CCD+1. The report is ALWAYS sent (the owner gets the dashboard of -failures + links either way). The step's OUTCOME is then decided by the UI gate: if the -UI-automation pass rate across both MRWP runs is >= RC_UI_PASS_THRESHOLD (90%) it -records `pass` and the flow advances to go_test; below the bar it records `attention` -(the step BLOCKS) so the owner investigates the large failure (usually a fix + MRWP -re-run) before proceeding. +failures + links either way). The step's OUTCOME is then decided by the three-tier UI +gate on the combined UI-automation pass rate across both MRWP runs: 100% is a clean +pass; >= RC_UI_PASS_THRESHOLD (90%) but < 100% passes with a warning (investigate the +failing tests in parallel — bug bash is NOT blocked); below 90% records `attention` +(the step BLOCKS) so the owner investigates the large failure and rules on it (patch + +re-trigger RC, or proceed as an automation flake). On a clean/warn pass the release +auto-advances into Phase 3 (bug bash). Sending email needs the WorkIQ MCP the engine can't reach, so this is a `scout` step: `build()` composes the email deterministically and returns a @@ -51,9 +53,14 @@ def build(state): return Blocked(f"rc_report: could not build the RC report ({e}).") gate = K.rc_ui_gate(model) - if gate["verdict"] == "pass": + v = gate["verdict"] + if v == "clean": summary = (f"Email the RC verification report to the release owner ({to}) — " - f"UI gate PASS") + f"UI gate CLEAN (100%)") + elif v == "warn": + summary = (f"Email the RC verification report to the release owner ({to}) — " + f"UI gate PASS with warning ({gate['pass_pct']}%); proceed + investigate " + f"failing UI tests in parallel") else: summary = (f"Email the RC verification report to the release owner ({to}) — " f"UI gate FAIL ({gate['pass_pct']}% < {int(K.RC_UI_PASS_THRESHOLD)}%); " diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 6624a055..c70edf4b 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -163,6 +163,15 @@ def _orch(signed=True): return st, orch +def _advance_to_first_gate(orch): + """Now that go_test is gone, the first real GATE is Phase-3 `bug_bash.bash_done`, + reached after the Phase-3 `ui_failures` human reminder. Drive to that reminder, clear + it, then drive to the bash_done gate.""" + orch.run_until_gate() # holds at ui_failures (reminder) + orch.complete_step("bug_bash", "ui_failures", "test: UI failures reviewed") + orch.run_until_gate() # holds at bash_done (gate) + + # ---- readiness entry gate ---- def test_entry_gate_blocks_before_signing(): @@ -185,10 +194,10 @@ def test_signing_clears_entry_gate(): _clear_phase0_scout(orch) _clear_ccd_scout(orch) orch.run_until_gate() - # Phase 0 and Phase 1 have no gate (branch cut is automatic); the first real - # gate is go_test at the end of Build & Lib Verification. - assert st.current_step == "go_test" - assert st.status == "holding_gate" + # Phases 0, 1 and 2 have no human gate (rc_report's 90% UI gate is auto); the first + # hold is the Phase-3 'ui_failures' human reminder. + assert st.current_step == "ui_failures" + assert st.status == "awaiting_action" def test_partial_sign_does_not_clear(): @@ -619,40 +628,40 @@ def test_render_is_consistent_and_has_links(): # ---- phase flow (readiness pre-signed) ---- -def test_holds_at_first_gate(): +def test_holds_at_first_hold(): st, orch = _orch() actions = orch.run_until_gate() - assert actions[-1].kind == "gate" - assert actions[-1].step == "go_test" # Phases 0 & 1 are gateless; first gate is go_test (Phase 2) - # auto steps that RUN before the first gate: Phase-0 breaking/cg/cron/wiki (4) + - # Phase-2 build_verify checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + - # rc_report (scout email, mocked done here) (1). The Phase-1 scout steps are - # pre-recorded by _orch's _clear_ccd_scout (not "ran"). - assert sum(1 for a in actions if a.kind == "ran") == 9 + assert actions[-1].kind == "reminder" + assert actions[-1].step == "ui_failures" # Phases 0-2 gateless (rc_report auto); first hold is Phase-3 ui_failures + # auto steps that RUN before the first hold: Phase-0 breaking/cg/cron/wiki (4) + + # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout + # email, mocked done here) (1) + Phase-3 clone_plans/coordinate stubs (2). The Phase-1 + # scout steps are pre-recorded by _orch's _clear_ccd_scout (not "ran"). + assert sum(1 for a in actions if a.kind == "ran") == 11 def test_gate_blocks_until_approved(): st, orch = _orch() - orch.run_until_gate() + _advance_to_first_gate(orch) assert st.status == "holding_gate" orch.run_until_gate() assert st.status == "holding_gate" - assert not st.is_done("build_verify", "go_test") + assert not st.is_done("bug_bash", "bash_done") def test_approve_advances(): st, orch = _orch() - orch.run_until_gate() + _advance_to_first_gate(orch) orch.approve_gate("ok") - assert st.is_done("build_verify", "go_test") + assert st.is_done("bug_bash", "bash_done") orch.run_until_gate() - assert st.current_step == "ui_failures" # next stop after go_test: a bug-bash human to-do - assert st.status == "awaiting_action" + assert st.current_step == "gate_watch" # next stop after bash_done: the Phase-4 finalize gate + assert st.status == "holding_gate" def test_deny_blocks(): st, orch = _orch() - orch.run_until_gate() + _advance_to_first_gate(orch) orch.deny_gate("flag not approved") assert st.status == "blocked" assert any("denied" in p for p in st.pending_human) @@ -684,16 +693,16 @@ def test_persistence_roundtrip(): orch.gate.sign() _clear_phase0_scout(orch) _clear_ccd_scout(orch) - orch.run_until_gate() + _advance_to_first_gate(orch) st.save(path) # reload — simulates resuming next day st2 = ReleaseState.load(path) assert st2.status == "holding_gate" - assert st2.current_step == "go_test" + assert st2.current_step == "bash_done" assert st2.readiness_signed # readiness survives the roundtrip orch2 = Orchestrator(CONFIG, st2) orch2.approve_gate("resumed") - assert st2.is_done("build_verify", "go_test") + assert st2.is_done("bug_bash", "bash_done") def test_conditional_hotfix_excluded_by_default(): @@ -713,31 +722,31 @@ def test_conditional_hotfix_excluded_by_default(): def test_skip_requires_reason(): st, orch = _orch() - orch.run_until_gate() # holds at go_test - act = orch.skip_step("build_verify", "go_test", "") # no reason + _advance_to_first_gate(orch) # holds at bash_done + act = orch.skip_step("bug_bash", "bash_done", "") # no reason assert act.kind == "idle" - assert not st.is_done("build_verify", "go_test") # unchanged + assert not st.is_done("bug_bash", "bash_done") # unchanged def test_skip_advances_past_gate(): st, orch = _orch() - orch.run_until_gate() - orch.skip_step("build_verify", "go_test", "n/a this release") - assert st.is_done("build_verify", "go_test") # skipped counts as done - rec = st.steps[st.key("build_verify", "go_test")] + _advance_to_first_gate(orch) + orch.skip_step("bug_bash", "bash_done", "n/a this release") + assert st.is_done("bug_bash", "bash_done") # skipped counts as done + rec = st.steps[st.key("bug_bash", "bash_done")] assert rec["status"] == "skipped" orch.run_until_gate() - assert st.current_step == "ui_failures" # advanced past the gate to the next hold + assert st.current_step == "gate_watch" # advanced past the gate to the Phase-4 gate def test_reopen_step(): st, orch = _orch() - orch.run_until_gate(); orch.approve_gate("ok") - assert st.is_done("build_verify", "go_test") - orch.reopen_step("build_verify", "go_test") - assert not st.is_done("build_verify", "go_test") # back to pending + _advance_to_first_gate(orch); orch.approve_gate("ok") + assert st.is_done("bug_bash", "bash_done") + orch.reopen_step("bug_bash", "bash_done") + assert not st.is_done("bug_bash", "bash_done") # back to pending orch.run_until_gate() - assert st.current_step == "go_test" # gate re-holds + assert st.current_step == "bash_done" # gate re-holds def test_halt_blocks_then_resume(): @@ -978,7 +987,7 @@ def test_no_anchor_when_ccd_unknown_runs_immediately(): """Backward-compatible: with no CCD stored, the anchor is inert and Phase 0 runs.""" st, orch = _orch() # ccd is None orch.run_until_gate() - assert st.status == "holding_gate" # reached flag_freeze, not 'scheduled' + assert st.status == "awaiting_action" # reached the first hold (Phase-3 ui_failures), not 'scheduled' # ---- reminder steps (human, non-gate → hold until done) ---- @@ -1079,11 +1088,11 @@ def test_digest_silent_while_scout_pending(): def test_notify_digest_reports_gate_and_progress(): from orchestrator import render st, orch = _orch() # signed, no CCD → phase due immediately - orch.run_until_gate() # Phases 0 & 1 gateless; holds at go_test (Phase 2) + orch.run_until_gate() # Phases 0-2 gateless; holds at the Phase-3 ui_failures action msg = render.notification(orch.status_report()) assert "Progress:" in msg - assert "Waiting on your decision" in msg and "proceed to bug bash" in msg.lower() - assert "your approval" in msg # lists the human touchpoint + assert "Action needed now" in msg # ui_failures is the live hold + assert "your approval" in msg # the bash_done gate is listed among the human touchpoints def test_notify_json_carries_owner_and_subject(): @@ -1976,19 +1985,20 @@ def test_build_verify_checker_blocks_when_not_triggered(): def test_build_verify_phase_shape(): - """Phase 2 has the 4 verification agent steps + the rc_report scout email + the - go_test human gate (in order), CCD+1 anchored, and the old action stubs are gone.""" + """Phase 2 has the 4 verification agent steps + the rc_report scout step (which emails + the RC report AND applies the 90% UI gate). rc_report is the terminal step — there is + NO separate human gate (the gate IS the decision). CCD+1 anchored.""" import yaml as _yaml cfg = _yaml.safe_load(open(CONFIG, encoding="utf-8")) bv = next(p for p in cfg["phases"] if p["id"] == "build_verify") ids = [s["id"] for s in bv["steps"]] assert ids == ["checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", - "rc_report", "go_test"] + "rc_report"] assert bv.get("anchor") == "CCD+1" rc = next(s for s in bv["steps"] if s["id"] == "rc_report") assert rc.get("source") == "scout" and rc.get("owner") == "agent" - gate = bv["steps"][-1] - assert gate["id"] == "go_test" and gate.get("gate") and gate.get("owner") == "human" + assert bv["steps"][-1]["id"] == "rc_report" # terminal Phase-2 step + assert not any(s.get("gate") for s in bv["steps"]) # no human gate in Phase 2 def test_build_verify_rc_report_emails_owner(): @@ -2040,9 +2050,10 @@ def test_build_verify_rc_report_emails_owner(): def test_rc_ui_gate_and_run_links(): - """The Phase-2 UI gate aggregates UI-automation results across BOTH MRWP providers: - >=90% combined pass → 'pass'; below → 'attention' (with a failing-suite summary); - no UI tests → pass with a warning. rc_run_links surfaces every evaluated run.""" + """The Phase-2 UI gate is three-tier on the combined UI pass rate across BOTH MRWP + providers: 100% → 'clean'; >=90% & <100% → 'warn' (non-blocking, investigate in + parallel); <90% → 'attention' (blocking, with a failing-suite summary); no UI tests → + 'clean' with a warning. rc_run_links surfaces every evaluated run.""" from steps.build_verify import _common as K def _model(ecs_ui, local_ui, ecs_suites=None): @@ -2056,24 +2067,32 @@ def _model(ecs_ui, local_ui, ecs_suites=None): "Local": {"run_id": 444, "failed_suites": [], "tests": {"categories": {"ui": local_ui}}}}} - # 180/200 = 90.0% → exactly at the bar → pass + # 200/200 = 100% → clean (non-blocking) + g0 = K.rc_ui_gate(_model({"total": 100, "passed": 100, "failed": 0}, + {"total": 100, "passed": 100, "failed": 0})) + assert g0["verdict"] == "clean" and g0["blocking"] is False and g0["pass_pct"] == 100.0 + + # 180/200 = 90.0% → exactly at the bar, not clean → warn (non-blocking) g = K.rc_ui_gate(_model({"total": 100, "passed": 100, "failed": 0}, {"total": 100, "passed": 80, "failed": 20})) - assert g["verdict"] == "pass" and g["pass_pct"] == 90.0 and g["ui_total"] == 200 + assert g["verdict"] == "warn" and g["blocking"] is False + assert g["pass_pct"] == 90.0 and g["ui_total"] == 200 + assert "in parallel" in g["detail"] - # 160/200 = 80% → below the bar → attention, with the failing suite listed + # 160/200 = 80% → below the bar → attention (blocking), with the failing suite listed fail_model = _model({"total": 100, "passed": 60, "failed": 40}, {"total": 100, "passed": 100, "failed": 0}, ecs_suites=[{"name": "PROD MSAL - RC Broker (API 32)", "failed": 40, "total": 100, "category": "ui"}]) g2 = K.rc_ui_gate(fail_model) - assert g2["verdict"] == "attention" and g2["pass_pct"] == 80.0 + assert g2["verdict"] == "attention" and g2["blocking"] is True and g2["pass_pct"] == 80.0 assert "BELOW" in g2["detail"] and "PROD MSAL - RC Broker (API 32)" in g2["detail"] - # no UI tests anywhere → pass with a warning (absence of data is not a failure) + # no UI tests anywhere → clean with a warning (absence of data is not a failure) g3 = K.rc_ui_gate({"mrwp": {"ECS": {"tests": {"categories": {}}}, "Local": {"tests": {"categories": {}}}}}) - assert g3["verdict"] == "pass" and g3["ui_total"] == 0 and "No UI-automation" in g3["detail"] + assert g3["verdict"] == "clean" and g3["blocking"] is False + assert g3["ui_total"] == 0 and "No UI-automation" in g3["detail"] # every evaluated run becomes a durable link links = K.rc_run_links(fail_model) @@ -2341,23 +2360,24 @@ def test_digest_shows_rc_line_when_build_verify_active(): def test_sim_fast_forwards_to_rc_gate_offline(): - """The at_rc_gate scenario (fine input mocks, no az) fast-forwards Phases 0-1, - runs the 4 build_verify steps for real on injected inputs, stashes the pipeline - ids, and halts at the go_test gate — all offline.""" + """The at_rc_gate scenario (fine input mocks, no az) fast-forwards Phases 0-1, runs the + 4 build_verify steps for real on injected inputs, stashes the pipeline ids, auto-advances + rc_report, and lands past Phase 2 at the bug-bash entry — all offline.""" import tempfile from orchestrator import sim as SIM with tempfile.TemporaryDirectory() as tmp: res = SIM.run_scenario("at_rc_gate", runs_root=tmp) - assert res.reached and res.stop_kind == "gate" + assert res.reached and res.stop_kind == "done" st = res.state # earlier phases complete assert all(st.is_done("preflight", s) for s in ("notice", "confirm_reminders", "vitals", "wiki")) assert all(st.is_done("ccd", s) for s in ("final_reminder", "localization")) - # the 4 verification steps ran (real build() on mocks) and are done - for s in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): + # the 4 verification steps ran (real build() on mocks) and rc_report auto-advanced + for s in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "rc_report"): assert st.is_done("build_verify", s), s - assert not st.is_done("build_verify", "go_test") # gate still holding + from orchestrator.engine import Orchestrator as _O + assert _O(CONFIG, st).current_phase_id() == "bug_bash" # positioned past Phase 2 # pipeline ids were stashed by the steps during the sim assert st.pipeline_runs.get("orchestrator") == "1678611" assert st.pipeline_runs.get("mrwp_ecs") == "1678863" @@ -2379,12 +2399,13 @@ def test_sim_open_positions_at_target_entry(): assert st.is_done("preflight", "wiki") and st.is_done("ccd", "localization") # nothing in the target phase has run assert not any(st.is_done("build_verify", s) for s in - ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "go_test")) + ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "rc_report")) assert st.current_phase == "build_verify" def test_sim_done_mode_completes_phase_and_advances(): - """`at: done` auto-approves the target's gate and lands at the next phase.""" + """`at: done` runs the whole target phase and lands at the next phase (Phase 2 has no + gate now — rc_report auto-advances).""" import tempfile from orchestrator import sim as SIM scenario = {"name": "t_done", "release_id": "2026-08", "ccd": "2026-08-26", @@ -2394,8 +2415,7 @@ def test_sim_done_mode_completes_phase_and_advances(): res = SIM.run_scenario(scenario, runs_root=tmp) st = res.state assert all(st.is_done("build_verify", s) for s in - ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "go_test")) - assert "build_verify.go_test" in res.gates_approved + ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "rc_report")) from orchestrator.engine import Orchestrator orch = Orchestrator(CONFIG, st) assert orch.current_phase_id() == "bug_bash" @@ -2855,8 +2875,8 @@ def test_ccd_steps_blocked_without_ccd(): def test_ccd_phase_shape_and_scout_kinds(): """Phase 1 is three scout comms/trigger steps and NO gate — the branch cut is - automatic (at 11 PM), so there's no manual cut step; the next gate is go_test in - Phase 2.""" + automatic (at 11 PM), so there's no manual cut step; the next hold is the Phase-3 + ui_failures reminder (Phase 2's rc_report gate is automatic).""" import yaml cfg = yaml.safe_load(open(CONFIG, encoding="utf-8")) ccd = next(p for p in cfg["phases"] if p["id"] == "ccd") @@ -3285,11 +3305,11 @@ def test_local_mock_completes_scout_step(): def test_local_mock_never_mocks_a_gate(): """Gate steps are not mockable — a gate still holds for a real decision even if someone lists it in the mock file.""" - st, orch = _mock_orch({"build_verify.go_test": {"outcome": "done"}}, as_of="2026-07-09") - _clear_phase0_scout(orch) # clear Phase-0 holds so we reach the Phase-2 gate + st, orch = _mock_orch({"bug_bash.bash_done": {"outcome": "done"}}, as_of="2026-07-09") + _clear_phase0_scout(orch) # clear Phase-0 holds _clear_ccd_scout(orch) # clear Phase-1 scout comms (Phase 1 is gateless) - orch.run_until_gate() - assert not st.is_done("build_verify", "go_test") + _advance_to_first_gate(orch) # Phases 0-2 gateless; clear ui_failures → hold at bash_done + assert not st.is_done("bug_bash", "bash_done") assert st.status == "holding_gate" From d124d90185d9dfb6cec2c75468d26caa07d7be15 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 20:24:04 +0100 Subject: [PATCH 66/90] release-agent: remove step-id coupling from the generic planner + render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design-audit follow-up to the go_test removal — eliminate the same 'step id hardcoded across modules' smell in two more places: 1) automations planner no longer special-cases ccd.localization. A step module may now declare automation_prompt(release, spec) (single source of truth, like fire_at_local); localization's bespoke trigger/poller prompts move into steps/ccd/localization.py and _prompt_for() delegates generically. 2) render/digest no longer hardcode the build_verify phase id for the RC pipeline-run line — driven by a data flag (show_pipeline_runs) in phases.yaml, propagated via engine _active_phase_report. Engine core confirmed free of hardcoded step ids. 181/181 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/phases.yaml | 1 + release-agent/orchestrator/automations.py | 61 +++++++---------------- release-agent/orchestrator/engine.py | 1 + release-agent/orchestrator/render.py | 7 +-- release-agent/steps/ccd/localization.py | 43 ++++++++++++++++ release-agent/tests/test_engine.py | 23 +++++++-- 6 files changed, 86 insertions(+), 50 deletions(-) diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index e07fcda3..41e4d664 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -48,6 +48,7 @@ phases: name: "Build & Lib Verification" checklist_phase: 2 anchor: "CCD+1" # opens the day AFTER Code Complete — the engineer wakes to a resume + show_pipeline_runs: true # surface the RC pipeline-run one-liner (checker/orchestrator/MRWP) in status + digest steps: - { id: checker_fired, name: "Verify Code Complete Checker fired the release", owner: agent, maps_to: [B0] } - { id: orchestrator_health, name: "Verify Release Orchestrator health (parked at Remove RC Tags)", owner: agent, maps_to: [B1] } diff --git a/release-agent/orchestrator/automations.py b/release-agent/orchestrator/automations.py index fc15de9a..bb70d070 100644 --- a/release-agent/orchestrator/automations.py +++ b/release-agent/orchestrator/automations.py @@ -138,50 +138,25 @@ def _ccd_cron(ccd_date, hhmm: str): def _prompt_for(spec: dict, release: str) -> str: """A concrete instruction the automation runs. Scout resolves each step via - step-action, executes the send/trigger, records it, and journals it. The - localization trigger and its poller have bespoke prompts (they don't just - record-step done).""" - step_list = ", ".join(spec["steps"]) - steps = spec.get("steps") or [] + step-action, executes the send/trigger, records it, and journals it. - # Localization poller (interval) — poll the in-flight run. - if spec.get("interval") and steps == ["ccd.localization"]: - return ( - f"Release {release} — localization poller.\n" - f"If localization for {release} is in-flight (it was triggered at noon and " - f"isn't done/blocked yet), poll it once. The ADO MCP can't reach " - f"msazure/One, so read via az (build id is stored on the step):\n" - f"1. status: `az pipelines build show --id " - f"--org https://msazure.visualstudio.com --project One " - f"--query \"{{status:status,result:result}}\" -o json`.\n" - f"2. if completed, find the OneLocBuild@3 log id: `az devops invoke " - f"--org https://msazure.visualstudio.com --area build --resource timeline " - f"--route-parameters project=One buildId= --api-version 7.1 " - f"--query \"records[?name=='OneLocBuild@3'].log.id | [0]\" -o tsv`, then read " - f"it: `az devops invoke --org https://msazure.visualstudio.com --area build " - f"--resource logs --route-parameters project=One buildId= " - f"logId= --api-version 7.1`.\n" - f"3. run `check-localization --release {release} --complete " - f"[--logs \"\"]`.\n" - f"4. act on the printed decision: `timeout` → send the given email; " - f"`complete_pr` → post the given chat message to the Code reviews chat; " - f"`wait`/`complete_none`/`not_started`/`already_final` → nothing to send.\n" - f"Silently journal: `journal --release {release} --source scout --kind " - f"automation --text \"localization-poller: \"`. Stay silent if " - f"there is nothing to do.") - - # Localization trigger (one-shot, noon) — trigger then hand off to the poller. - if not spec.get("interval") and steps == ["ccd.localization"]: - return ( - f"Release {release} — trigger localization.\n" - f"1. run `step-action --release {release} --phase ccd --step localization`;\n" - f"2. run the returned needs_skill action to start pipeline 405133 " - f"(isCreatePrSelected=true); note the queued build id;\n" - f"3. run `record-localization-run --release {release} --build-id ` " - f"— this leaves the step IN-FLIGHT (do NOT record-step done; the poller " - f"finishes it once the run completes or times out);\n" - f"4. silently journal: `journal --release {release} --source scout --kind " - f"automation --text \"ccd-noon triggered ccd.localization\"`.") + A step MAY OWN a bespoke prompt by declaring `automation_prompt(release, spec)` on its + module (the single source of truth, like `fire_at_local`) — used for genuinely bespoke + flows such as the localization trigger + poller. This keeps the planner generic: it + never special-cases a step id. Steps without one get the default send + record-step + prompt below.""" + steps = spec.get("steps") or [] + step_list = ", ".join(steps) + + # Single-step automation whose step owns a bespoke prompt → delegate to the module. + if len(steps) == 1: + phase, _, sid = steps[0].partition(".") + mod = steps_pkg.get_step(phase, sid) + fn = getattr(mod, "automation_prompt", None) + if callable(fn): + prompt = fn(release, spec) + if prompt: + return prompt # Default: send/trigger + record-step done (reminders). return ( diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 567e0748..868f53d5 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -655,6 +655,7 @@ def _active_phase_report(self) -> Optional[dict]: return { "id": phase["id"], "name": phase["name"], "num": phase.get("checklist_phase"), + "show_pipeline_runs": bool(phase.get("show_pipeline_runs")), "done": done, "total": len(steps), "due": self._phase_due(phase), "started": done > 0, "opens": opens.isoformat() if opens else None, diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 70b21410..fe383f18 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -447,9 +447,10 @@ def _digest_model(r: dict): hold = ("action", r["action"]["step_name"]) human_all = [o for o in ap.get("outstanding", []) if o["gate"] or o["reminder"]] completed_all = ap.get("completed") or [] - # Phase-2 RC one-liner — only while build_verify is the active phase, best-effort - # (reads state.pipeline_runs; never a live call). Empty until the chain resolves. - rc_line = _pipelines_line(r) if ap.get("id") == "build_verify" else "" + # Phase-2 RC one-liner — only while a phase that opts in (show_pipeline_runs) is + # active, best-effort (reads state.pipeline_runs; never a live call). Empty until the + # chain resolves. + rc_line = _pipelines_line(r) if ap.get("show_pipeline_runs") else "" return { "rid": r.get("release_id", "?"), "ap": ap, diff --git a/release-agent/steps/ccd/localization.py b/release-agent/steps/ccd/localization.py index 1bf3771a..9e61faf1 100644 --- a/release-agent/steps/ccd/localization.py +++ b/release-agent/steps/ccd/localization.py @@ -341,6 +341,49 @@ def build(state): ) +def automation_prompt(release: str, spec: dict) -> str: + """The bespoke automation instruction for THIS step — owned here (single source of + truth, like `fire_at_local`) so the generic automations planner doesn't special-case + the step id. Two shapes: the interval POLLER vs the one-shot noon TRIGGER (the planner + passes the automation `spec`; `interval` set ⇒ poller).""" + if spec.get("interval"): + return ( + f"Release {release} — localization poller.\n" + f"If localization for {release} is in-flight (it was triggered at noon and " + f"isn't done/blocked yet), poll it once. The ADO MCP can't reach " + f"msazure/One, so read via az (build id is stored on the step):\n" + f"1. status: `az pipelines build show --id " + f"--org https://msazure.visualstudio.com --project One " + f"--query \"{{status:status,result:result}}\" -o json`.\n" + f"2. if completed, find the OneLocBuild@3 log id: `az devops invoke " + f"--org https://msazure.visualstudio.com --area build --resource timeline " + f"--route-parameters project=One buildId= --api-version 7.1 " + f"--query \"records[?name=='OneLocBuild@3'].log.id | [0]\" -o tsv`, then read " + f"it: `az devops invoke --org https://msazure.visualstudio.com --area build " + f"--resource logs --route-parameters project=One buildId= " + f"logId= --api-version 7.1`.\n" + f"3. run `check-localization --release {release} --complete " + f"[--logs \"\"]`.\n" + f"4. act on the printed decision: `timeout` → send the given email; " + f"`complete_pr` → post the given chat message to the Code reviews chat; " + f"`wait`/`complete_none`/`not_started`/`already_final` → nothing to send.\n" + f"Silently journal: `journal --release {release} --source scout --kind " + f"automation --text \"localization-poller: \"`. Stay silent if " + f"there is nothing to do.") + + # One-shot (noon) trigger — trigger then hand off to the poller. + return ( + f"Release {release} — trigger localization.\n" + f"1. run `step-action --release {release} --phase ccd --step localization`;\n" + f"2. run the returned needs_skill action to start pipeline 405133 " + f"(isCreatePrSelected=true); note the queued build id;\n" + f"3. run `record-localization-run --release {release} --build-id ` " + f"— this leaves the step IN-FLIGHT (do NOT record-step done; the poller " + f"finishes it once the run completes or times out);\n" + f"4. silently journal: `journal --release {release} --source scout --kind " + f"automation --text \"ccd-noon triggered ccd.localization\"`.") + + KNOWLEDGE = { "summary": "Trigger the loc pipeline at noon, poll it to completion, then post the translations PR for review.", "what": ( diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index c70edf4b..d942dd1b 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -1453,6 +1453,19 @@ def sync(): assert not u1["ccd-localization-poller"]["changed"] +def test_automation_prompt_delegates_to_step_module(): + """The planner is generic: a step that declares `automation_prompt` owns its bespoke + instruction (localization trigger vs poller), and steps without one get the default + send + record-step prompt — no step id is special-cased in automations.py.""" + from orchestrator import automations as A + by = {a["slug"]: a for a in A.plan(CONFIG, "2026-09", "2026-09-09")["automations"]} + # localization's module owns both bespoke prompts (delegated, not hardcoded here) + assert "trigger localization" in by["ccd-noon"]["prompt"] + assert "localization poller" in by["ccd-localization-poller"]["prompt"] + # a plain multi-step reminder automation uses the generic default prompt + assert "For EACH of these steps in order" in by["ccd-morning"]["prompt"] + + def test_ccd_cron_pins_to_exact_date(): """_ccd_cron builds a cron 'M H D Mo *' targeting the CCD's day+month+time, so a one-shot fires ON the CCD — never the next matching weekday (the early-fire bug).""" @@ -2340,11 +2353,12 @@ def test_build_verify_persists_pipeline_run_ids(): def test_digest_shows_rc_line_when_build_verify_active(): - """When Phase 2 (build_verify) is the active phase and run ids are on state, the daily - digest carries a one-line RC summary; other phases don't show it.""" + """When a phase that opts in (show_pipeline_runs) is active and run ids are on state, + the daily digest carries a one-line RC summary; phases that don't opt in omit it.""" from orchestrator import render r = {"release_id": "2026-08", "readiness_signed": True, "active_phase": {"id": "build_verify", "name": "Build & RC", "num": 2, + "show_pipeline_runs": True, "due": True, "started": True, "done": 2, "total": 5, "outstanding": [], "completed": ["checker_fired", "orchestrator_health"]}, "pipeline_runs": {"checker": "1678599", "orchestrator": "1678611", @@ -2354,8 +2368,9 @@ def test_digest_shows_rc_line_when_build_verify_active(): md = render.notification_markdown(r) assert "RC pipelines:" in text and "orchestrator 1678611" in text assert "MRWP ECS 900001 / Local 900002" in md - # a non-build_verify active phase omits the RC line - r2 = dict(r, active_phase=dict(r["active_phase"], id="prep", name="Prep")) + # a phase that doesn't opt in omits the RC line + r2 = dict(r, active_phase=dict(r["active_phase"], id="prep", name="Prep", + show_pipeline_runs=False)) assert "RC pipelines:" not in render.notification(r2) From 0c612f09ebbee76a12ff975830bf9772d9bee452 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 21:30:55 +0100 Subject: [PATCH 67/90] release-agent: store RC pipeline runs in state (nested schema) + reuse in the report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rc_report + the 90% UI gate no longer re-discover pipeline ids live — they read the RECORD the verification steps store in state.pipeline_runs. Schema: { checker{run_id,when}, orchestrator{run_id,versions{},parked}, rcs:[{rc,ecs{...},local{...}}] } — 1 checker + 1 orchestrator, N RC iterations (a re-trigger appends a new ecs/local pair), latest = rcs[-1]. Each provider slot snapshots stage completion + the Test-tab summary + failing suites, so the gate is deterministic/replayable from state alone. Legacy flat pipeline_runs auto-migrates to the nested shape on ReleaseState.load. The live rc-report diagnostic still reads fresh and refreshes the record. render._pipelines_line reads the nested shape (latest RC). 183 tests pass (incl. migration + append-new-rc coverage). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../orchestrator/commands/rc_report.py | 32 ++- release-agent/orchestrator/render.py | 37 +-- release-agent/orchestrator/state.py | 66 ++++- release-agent/steps/build_verify/_common.py | 136 ++++++++-- .../steps/build_verify/checker_fired.py | 2 +- release-agent/steps/build_verify/mrwp_ecs.py | 3 +- .../steps/build_verify/mrwp_local.py | 3 +- .../steps/build_verify/orchestrator_health.py | 6 +- release-agent/tests/test_engine.py | 247 +++++++++++------- 9 files changed, 381 insertions(+), 151 deletions(-) diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index d641021d..5e9b1908 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -27,22 +27,28 @@ def cmd_rc_report(args): def _persist(st, model, args): - """Record the resolved run ids on state so status/digest can show them without a - live read. Best-effort — a report must never fail because the state write did.""" + """Record the resolved runs (+ snapshots) on state so status/digest/rc_report read + them without a live call. Best-effort — a report must never fail because the state + write did. (This is the LIVE `rc-report` diagnostic refreshing the record; the verify + steps are the primary writers.)""" if st is None: return - ch = model.get("checker") or {} - o = model.get("orchestrator") or {} - mr = model.get("mrwp") or {} - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or None try: - K.stash_runs(st, - checker=ch.get("run_id"), - orchestrator=o.get("run_id"), - versions=vstr, - mrwp_ecs=(mr.get("ECS") or {}).get("run_id"), - mrwp_local=(mr.get("Local") or {}).get("run_id")) + ch = model.get("checker") or {} + if ch.get("run_id"): + K.stash_checker(st, ch["run_id"], ch.get("when")) + o = model.get("orchestrator") or {} + if o.get("run_id"): + K.stash_orchestrator(st, o["run_id"], + versions={k: v for k, v in (o.get("versions") or {}).items() if v}, + parked=o.get("parked")) + mr = model.get("mrwp") or {} + for slot in ("ECS", "Local"): + m = mr.get(slot) or {} + if m.get("run_id"): + K.stash_mrwp(st, slot, {k: m.get(k) for k in + ("run_id", "complete", "ran", "total", "failed_stages", + "yellow_stages", "never_ran", "tests", "failed_suites")}) C.save_state(st, args.runs_root, args.release) except Exception: pass diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index fe383f18..7e30208f 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -237,24 +237,31 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: def _pipelines_line(r: dict) -> str: - """Compact one-line summary of the Phase-2 release-pipeline run ids recorded on - state (checker → orchestrator → the two MRWP runs). Empty string when none resolved - yet. Shared by the status view and the daily digest so both read from state (no live - az call in the render path).""" - pr = r.get("pipeline_runs") or {} + """Compact one-line summary of the Phase-2 release-pipeline runs recorded on state + (checker → orchestrator → the LATEST RC's two MRWP runs). Empty string when none + resolved yet. Reads the nested pipeline_runs schema (migrating a legacy flat shape); + no live az call in the render path.""" + from orchestrator.state import migrate_pipeline_runs + pr = migrate_pipeline_runs(r.get("pipeline_runs") or {}) parts = [] - if pr.get("checker"): - parts.append(f"checker {pr['checker']}") - if pr.get("orchestrator"): - v = f" ({pr['versions']})" if pr.get("versions") else "" - parts.append(f"orchestrator {pr['orchestrator']}{v}") + ch = pr.get("checker") or {} + if ch.get("run_id"): + parts.append(f"checker {ch['run_id']}") + o = pr.get("orchestrator") or {} + if o.get("run_id"): + v = o.get("versions") or {} + vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) + parts.append(f"orchestrator {o['run_id']}" + (f" ({vstr})" if vstr else "")) + rcs = pr.get("rcs") or [] + rc = rcs[-1] if rcs else {} mr = [] - if pr.get("mrwp_ecs"): - mr.append(f"ECS {pr['mrwp_ecs']}") - if pr.get("mrwp_local"): - mr.append(f"Local {pr['mrwp_local']}") + if (rc.get("ecs") or {}).get("run_id"): + mr.append(f"ECS {rc['ecs']['run_id']}") + if (rc.get("local") or {}).get("run_id"): + mr.append(f"Local {rc['local']['run_id']}") if mr: - parts.append("MRWP " + " / ".join(mr)) + tag = f" (RC{rc['rc']})" if rc.get("rc") and len(rcs) > 1 else "" + parts.append("MRWP" + tag + " " + " / ".join(mr)) return " · ".join(parts) diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py index 8e1301c5..5a9389e1 100644 --- a/release-agent/orchestrator/state.py +++ b/release-agent/orchestrator/state.py @@ -23,6 +23,54 @@ def _now() -> str: return datetime.now(timezone.utc).isoformat() +def _versions_str_to_dict(v) -> dict: + """'Common 24.6.0, Msal 8.4.2, Broker 16.5.0' -> {Common,Msal,Broker}. A dict passes + through; anything unparseable yields {}.""" + if isinstance(v, dict): + return {k: val for k, val in v.items() if val} + if not isinstance(v, str) or not v.strip(): + return {} + out = {} + for part in v.split(","): + toks = part.strip().split() + if len(toks) >= 2: + out[toks[0]] = toks[1] + return out + + +def migrate_pipeline_runs(pr) -> dict: + """Normalize the pipeline_runs container to the nested RC schema (idempotent). + + Accepts the legacy FLAT shape + {checker, orchestrator, versions, mrwp_ecs, mrwp_local, mrwp_id_source, resolved_at} + and lifts it to + {checker:{run_id,...}, orchestrator:{run_id,versions:{},...}, rcs:[{rc:1,ecs,local}]}. + A value already in the nested shape (has 'rcs', or a dict 'checker') is returned as-is. + Empty/None -> {}.""" + if not pr or not isinstance(pr, dict): + return {} + # Already nested? (rcs present, or checker is an object) + if "rcs" in pr or isinstance(pr.get("checker"), dict) or isinstance(pr.get("orchestrator"), dict): + return pr + out = {} + if pr.get("checker"): + out["checker"] = {"run_id": str(pr["checker"]), "resolved_at": pr.get("resolved_at")} + if pr.get("orchestrator"): + out["orchestrator"] = {"run_id": str(pr["orchestrator"]), + "versions": _versions_str_to_dict(pr.get("versions")), + "resolved_at": pr.get("resolved_at")} + ecs, local = pr.get("mrwp_ecs"), pr.get("mrwp_local") + if ecs or local: + rc = {"rc": 1, "resolved_at": pr.get("resolved_at")} + src = pr.get("mrwp_id_source") + if ecs: + rc["ecs"] = {"run_id": str(ecs), "id_source": src} + if local: + rc["local"] = {"run_id": str(local), "id_source": src} + out["rcs"] = [rc] + return out + + @dataclass class StepState: """Persisted state for a single step.""" @@ -88,7 +136,17 @@ class ReleaseState: # refreshed each time Phase 2 resolves the chain. Because a re-triggered 'Trigger RC # Testing' stage spawns NEW MRWP runs, these are re-resolved (newest wins) — not a # fixed cache. Surfaced in status details + the daily digest. - pipeline_runs: dict = field(default_factory=dict) # {checker, orchestrator, mrwp_ecs, mrwp_local, mrwp_id_source, resolved_at} + # Phase-2 release-pipeline runs — the RECORD of what verification resolved, reused by + # the RC report + gate (no re-discovery). Nested schema (see migrate_pipeline_runs): + # { checker: {run_id, when, resolved_at}, + # orchestrator: {run_id, versions:{Common,Msal,Broker}, parked, resolved_at}, + # rcs: [ {rc, ecs:{run_id,id_source,complete,ran,total,failed_stages, + # yellow_stages,never_ran,tests,failed_suites,resolved_at}, + # local:{...same...}, resolved_at} ] } + # There is exactly ONE checker + ONE orchestrator, but MULTIPLE RC iterations (each a + # re-trigger of RC Testing spawns a new ecs/local pair). The LATEST RC is rcs[-1] — the + # report + gate always use it. A per-provider id change appends a new rc entry. + pipeline_runs: dict = field(default_factory=dict) notes: list = field(default_factory=list) # ---- persistence ---- @@ -101,7 +159,11 @@ def load(cls, path: str) -> "ReleaseState": # unattended automation — an unexpected key must never hard-crash it. # Only keys matching a declared field are applied; the rest are dropped. known = {f.name for f in fields(cls)} - return cls(**{k: v for k, v in data.items() if k in known}) + obj = cls(**{k: v for k, v in data.items() if k in known}) + # Migrate the flat pre-nested pipeline_runs shape to the nested RC schema so + # older/hand-edited state files load cleanly and readers see one shape. + obj.pipeline_runs = migrate_pipeline_runs(obj.pipeline_runs) + return obj def save(self, path: str) -> None: self.updated_at = _now() diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 629adf50..ce28237c 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -45,17 +45,64 @@ def links_for(build_id, name="ADO run"): return [{"name": name, "url": build_url(build_id)}] -def stash_runs(state, **ids): - """Record resolved pipeline run ids on state.pipeline_runs (drop None values), - stamped with resolved_at. Called each time Phase 2 resolves the chain so the ids - are in state (for status details + the digest). Re-resolved on every pass, so a - re-triggered MRWP run (new id) overwrites the old one.""" +def _now_iso(): from datetime import datetime, timezone - pr = dict(getattr(state, "pipeline_runs", {}) or {}) - for k, v in ids.items(): - if v is not None: - pr[k] = str(v) - pr["resolved_at"] = datetime.now(timezone.utc).isoformat() + return datetime.now(timezone.utc).isoformat() + + +def _pipeline_runs(state) -> dict: + """The nested pipeline_runs container on state (migrating a legacy flat shape).""" + from orchestrator.state import migrate_pipeline_runs + return migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + + +def stash_checker(state, run_id, when=None): + """Record the (single) Code Complete Checker run that fired the release.""" + pr = _pipeline_runs(state) + pr["checker"] = {"run_id": str(run_id), "when": when, "resolved_at": _now_iso()} + state.pipeline_runs = pr + + +def stash_orchestrator(state, run_id, versions=None, parked=None): + """Record the (single) Release Orchestrator run + its RC versions and parked flag.""" + pr = _pipeline_runs(state) + pr["orchestrator"] = {"run_id": str(run_id), + "versions": versions or {}, + "parked": parked, + "resolved_at": _now_iso()} + state.pipeline_runs = pr + + +def latest_rc(state) -> dict: + """The current RC iteration (the last entry in rcs), or {} when none resolved yet.""" + rcs = _pipeline_runs(state).get("rcs") or [] + return rcs[-1] if rcs else {} + + +def stash_mrwp(state, provider, snapshot): + """Record an MRWP provider run's FULL verification snapshot into the current RC + iteration. `provider` is 'ECS' or 'Local'; `snapshot` carries run_id + stage/test + results (run_id, id_source, complete, ran, total, failed_stages, yellow_stages, + never_ran, tests, failed_suites). + + RC iterations are a list; the LATEST is rcs[-1]. When this provider's slot in the + current RC already holds a DIFFERENT run_id, RC Testing was re-triggered → a NEW rc + entry is appended (rc = last+1) and the snapshot lands there. Same id → idempotent + update. So ecs/local resolving in separate steps (any order) merge into one rc, and a + re-trigger rolls forward to the next rc.""" + key = provider.lower() # 'ecs' | 'local' + pr = _pipeline_runs(state) + rcs = pr.setdefault("rcs", []) + cur = rcs[-1] if rcs else None + existing = (cur or {}).get(key) or {} + if cur is None or (existing.get("run_id") and existing["run_id"] != str(snapshot.get("run_id"))): + cur = {"rc": (rcs[-1]["rc"] + 1) if rcs else 1} + rcs.append(cur) + snap = dict(snapshot) + snap["run_id"] = str(snapshot.get("run_id")) + snap["resolved_at"] = _now_iso() + cur[key] = snap + cur["resolved_at"] = _now_iso() state.pipeline_runs = pr @@ -67,11 +114,46 @@ def stash_runs(state, **ids): # ---------------------------------------------------------------- RC report email def rc_report_model(state, timeout=120): - """The full Phase-2 RC report model (checker → orchestrator → ECS/Local MRWP + - per-run test breakdown) for this release. Pure read; see tools.pipelines.""" - from tools import pipelines as P - return P.release_report(ORG, PROJECT, state.release_id, - checker_def=CHECKER_DEF, orch_def=ORCHESTRATOR_DEF, timeout=timeout) + """The Phase-2 RC report model — assembled from the RECORD in state.pipeline_runs + (the verification steps stored it), NOT a live re-discovery. Uses the LATEST RC + iteration (rcs[-1]). Shape mirrors tools.pipelines.release_report so the gate + email + builders consume it unchanged: + {release, checker{fired,run_id,when}, orchestrator{found,healthy,parked,run_id,versions}, + mrwp{ECS{...}, Local{...}}, problems[], rc} + """ + from orchestrator.state import migrate_pipeline_runs + pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + ch = pr.get("checker") or {} + o = pr.get("orchestrator") or {} + rcs = pr.get("rcs") or [] + rc = rcs[-1] if rcs else {} + + model = { + "release": state.release_id, + "checker": {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), + "when": ch.get("when")}, + "orchestrator": {"found": bool(o.get("run_id")), "healthy": True, + "run_id": o.get("run_id"), "versions": o.get("versions") or {}, + "parked": o.get("parked")}, + "mrwp": {}, "problems": [], "rc": rc.get("rc"), + } + for slot, prov in (("ecs", "ECS"), ("local", "Local")): + s = rc.get(slot) + if not s: + continue + model["mrwp"][prov] = { + "run_id": s.get("run_id"), "complete": s.get("complete"), + "ran": s.get("ran"), "total": s.get("total"), + "failed_stages": s.get("failed_stages") or [], + "yellow_stages": s.get("yellow_stages") or [], + "never_ran": s.get("never_ran") or [], + "tests": s.get("tests"), "failed_suites": s.get("failed_suites"), + } + if not s.get("complete") and s.get("never_ran"): + model["problems"].append( + f"MRWP {prov}: did NOT run to completion — never-ran: " + f"{', '.join(n for n in s['never_ran'] if n)}.") + return model def rc_run_links(model) -> list: @@ -481,7 +563,8 @@ def verify_mrwp(state, provider): # 3) test summary (best-effort — never blocks; red/yellow tests are triaged later) tests = mock_input("tests", MISSING) - if tests is MISSING: + tests_injected = tests is not MISSING + if not tests_injected: ok, tests, _ = P.get_test_summary(ORG, PROJECT, mid) if not ok: tests = None @@ -495,6 +578,25 @@ def verify_mrwp(state, provider): if comp["yellow"]: extras.append(f"{len(comp['yellow'])} yellow") extra = f" ({', '.join(extras)} — triaged later)" if extras else "" - stash_runs(state, **{f"mrwp_{provider.lower()}": mid}) + + # 4) failing suites (individual test names) — snapshot alongside the summary so the RC + # report + gate read everything from state (no re-discovery). Mockable via `suites`. + # Only fetch LIVE when the summary was read live (tests not injected) — an injected + # summary means an offline/test context, so we don't make the extra network call. + suites = mock_input("suites", MISSING) + if suites is MISSING: + suites = None + if not tests_injected and tests and tests.get("failed"): + okf, fsuites, _ = P.get_failed_tests(ORG, PROJECT, mid) + if okf: + suites = fsuites + + # 5) stash the FULL per-provider snapshot into the current RC iteration. + stash_mrwp(state, provider, { + "run_id": mid, "complete": comp["complete"], "ran": comp["ran"], + "total": comp["total"], "failed_stages": comp["failed"], + "yellow_stages": comp["yellow"], "never_ran": comp["never_ran"], + "tests": tests, "failed_suites": suites, + }) return Done( f"{label} run {mid} ran to completion — {stage_note}{extra}.{tnote}", links=links) diff --git a/release-agent/steps/build_verify/checker_fired.py b/release-agent/steps/build_verify/checker_fired.py index 53bde2f5..081e9b51 100644 --- a/release-agent/steps/build_verify/checker_fired.py +++ b/release-agent/steps/build_verify/checker_fired.py @@ -89,7 +89,7 @@ def _verdict(state, run, result, job): return Blocked( f"Code Complete Checker '{job}' did not succeed (result={result}) in run " f"{bid} ({when}) — the orchestrator was not launched.{K.UNBLOCK_HELP}", links=links) - K.stash_runs(state, checker=bid) + K.stash_checker(state, bid, when) return Done( f"Code Complete Checker fired the release — run {bid} ({when}), '{job}' succeeded.", links=links) diff --git a/release-agent/steps/build_verify/mrwp_ecs.py b/release-agent/steps/build_verify/mrwp_ecs.py index 4fb8d24e..1720466e 100644 --- a/release-agent/steps/build_verify/mrwp_ecs.py +++ b/release-agent/steps/build_verify/mrwp_ecs.py @@ -19,7 +19,8 @@ MOCKABLE = { "mrwp_id": {"kind": "input", "desc": "Inject the ECS MRWP build id (skip orchestrator lookup)."}, "stages": {"kind": "input", "desc": "Inject the ECS run's stage list [{name,state,result}]."}, - "tests": {"kind": "input", "desc": "Inject the ECS test summary {total,passed,failed}."}, + "tests": {"kind": "input", "desc": "Inject the ECS test summary {total,passed,failed,categories}."}, + "suites": {"kind": "input", "desc": "Inject the ECS failing suites [{name,failed,total,category,tests}]."}, } diff --git a/release-agent/steps/build_verify/mrwp_local.py b/release-agent/steps/build_verify/mrwp_local.py index befcc098..972b29be 100644 --- a/release-agent/steps/build_verify/mrwp_local.py +++ b/release-agent/steps/build_verify/mrwp_local.py @@ -17,7 +17,8 @@ MOCKABLE = { "mrwp_id": {"kind": "input", "desc": "Inject the Local MRWP build id (skip orchestrator lookup)."}, "stages": {"kind": "input", "desc": "Inject the Local run's stage list [{name,state,result}]."}, - "tests": {"kind": "input", "desc": "Inject the Local test summary {total,passed,failed}."}, + "tests": {"kind": "input", "desc": "Inject the Local test summary {total,passed,failed,categories}."}, + "suites": {"kind": "input", "desc": "Inject the Local failing suites [{name,failed,total,category,tests}]."}, } diff --git a/release-agent/steps/build_verify/orchestrator_health.py b/release-agent/steps/build_verify/orchestrator_health.py index 84393c9f..82a079c2 100644 --- a/release-agent/steps/build_verify/orchestrator_health.py +++ b/release-agent/steps/build_verify/orchestrator_health.py @@ -82,9 +82,11 @@ def build(state): # The park stage should be PENDING (waiting for the owner). If it already ran, the # gate was approved — surface it (out of the expected Phase-2 state), don't hard-fail. - K.stash_runs(state, orchestrator=bid, versions=vstr) park = by_name.get(cfg["park_stage"]) - if park is not None and park.get("state") == "completed": + park_done = bool(park is not None and park.get("state") == "completed") + K.stash_orchestrator(state, bid, versions={k: v for k, v in versions.items() if v}, + parked=not park_done) + if park_done: return Done( f"Release Orchestrator run {bid} healthy ({vstr}); NOTE '{cfg['park_stage']}' " f"already ran (result={park.get('result')}) — the approval gate was cleared. " diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index d942dd1b..2ee12a94 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -1951,6 +1951,26 @@ def _bv_build(orch, st, sid): return as_dict(_steps.get_step("build_verify", sid).build(st)) +def _seed_rc_pipeline(st, ecs_ui, local_ui, *, ecs_suites=None, + ecs_id="1678863", local_id="1678864"): + """Seed state.pipeline_runs with a full RC snapshot (checker + orchestrator + one RC + pair) the way the verify steps would, so rc_report / record-rc-report read it from + state (no live re-discovery). `ecs_ui`/`local_ui` are the UI category dicts the gate + consumes ({total,passed,failed}).""" + from steps.build_verify import _common as K + K.stash_checker(st, "1678599", "2026-08-13T06:00") + K.stash_orchestrator(st, "1678611", + versions={"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}, + parked=True) + + def snap(run_id, ui, suites): + return {"run_id": run_id, "complete": True, "ran": 23, "total": 23, + "failed_stages": [], "yellow_stages": [], "never_ran": [], + "tests": {"categories": {"ui": ui}}, "failed_suites": suites or []} + K.stash_mrwp(st, "ECS", snap(ecs_id, ecs_ui, ecs_suites)) + K.stash_mrwp(st, "Local", snap(local_id, local_ui, None)) + + def test_build_verify_steps_pass_with_healthy_mocks(): """With injected healthy inputs, all four build_verify agent steps return done — and mrwp steps surface the test summary + red/yellow counts in the note.""" @@ -2016,50 +2036,49 @@ def test_build_verify_phase_shape(): def test_build_verify_rc_report_emails_owner(): """rc_report composes the RC report email to the release owner as a - NeedsSkill(workiq_send_email); blocks when no owner email is set. release_report is - monkeypatched so it's offline.""" + NeedsSkill(workiq_send_email) from the RECORD in state.pipeline_runs (no live call); + blocks when no owner email is set.""" from orchestrator.outcomes import as_dict - from tools import pipelines as P + from steps.build_verify import _common as K import steps as _steps - orig = P.release_report - P.release_report = lambda *a, **k: { - "release": "2026-08", "checker": {"fired": True, "run_id": 1678599}, - "orchestrator": {"found": True, "healthy": True, "parked": True, "run_id": 1678611, - "versions": {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}}, - "mrwp": {"ECS": {"run_id": 1678863, "complete": True, "ran": 23, "total": 23, - "failed_stages": ["UI Automation"], - "tests": {"total": 5871, "passed": 5767, "failed": 104, "runs": [], - "categories": { - "unit": {"total": 5248, "passed": 5248, "failed": 0}, - "instrumented": {"total": 442, "passed": 440, "failed": 2}, - "ui": {"total": 165, "passed": 63, "failed": 102}}}, - "failed_suites": [{"name": "PROD MSAL - RC Broker (API 32)", - "failed": 18, "total": 44, "category": "ui", - "tests": ["test_1_Foo", "test_2_Bar"]}]}, - "Local": {"run_id": 1678864, "complete": True, "ran": 23, "total": 23, - "failed_stages": [], "tests": {"total": 5856, "passed": 5756, "failed": 100, - "runs": [], "categories": {}}, - "failed_suites": []}}, - "problems": []} - try: - st = ReleaseState(release_id="2026-08", ccd="2026-08-26", - owner_email="dev@microsoft.com", owner_name="Dev") - out = as_dict(_steps.get_step("build_verify", "rc_report").build(st)) - assert out["kind"] == "needs_skill" and out["tool"] == "workiq_send_email" - assert out["payload"]["to"] == ["dev@microsoft.com"] and out["payload"]["isHtml"] - body = out["payload"]["body"] - assert "1678863" in body # run id present - assert "UI-automation failure rate" in body # per-category headline metric - assert "61.8%" in body # 102/165 UI failures — the real UI rate - assert "Unit" in body and "Instrumented" in body and "UI automation" in body - assert "test_1_Foo" in body # failing test names still listed - assert out["record_as"] == "rc_report" and out["outbound"] is True - # no owner → blocked - st2 = ReleaseState(release_id="2026-08", ccd="2026-08-26") - out2 = as_dict(_steps.get_step("build_verify", "rc_report").build(st2)) - assert out2["kind"] == "blocked" and "owner" in out2["reason"] - finally: - P.release_report = orig + + st = ReleaseState(release_id="2026-08", ccd="2026-08-26", + owner_email="dev@microsoft.com", owner_name="Dev") + # Seed the RC snapshot the verify steps would have stored (full categories + a suite). + K.stash_checker(st, "1678599", "2026-08-13T06:00") + K.stash_orchestrator(st, "1678611", + versions={"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}, + parked=True) + K.stash_mrwp(st, "ECS", { + "run_id": "1678863", "complete": True, "ran": 23, "total": 23, + "failed_stages": ["UI Automation"], "yellow_stages": [], "never_ran": [], + "tests": {"total": 5871, "passed": 5767, "failed": 104, "categories": { + "unit": {"total": 5248, "passed": 5248, "failed": 0}, + "instrumented": {"total": 442, "passed": 440, "failed": 2}, + "ui": {"total": 165, "passed": 63, "failed": 102}}}, + "failed_suites": [{"name": "PROD MSAL - RC Broker (API 32)", "failed": 18, + "total": 44, "category": "ui", "tests": ["test_1_Foo", "test_2_Bar"]}]}) + K.stash_mrwp(st, "Local", { + "run_id": "1678864", "complete": True, "ran": 23, "total": 23, + "failed_stages": [], "yellow_stages": [], "never_ran": [], + "tests": {"total": 5856, "passed": 5756, "failed": 100, "categories": {}}, + "failed_suites": []}) + + out = as_dict(_steps.get_step("build_verify", "rc_report").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "workiq_send_email" + assert out["payload"]["to"] == ["dev@microsoft.com"] and out["payload"]["isHtml"] + assert out["payload"]["followup_command"] == "record-rc-report" + body = out["payload"]["body"] + assert "1678863" in body # run id present + assert "UI-automation failure rate" in body # per-category headline metric + assert "61.8%" in body # 102/165 UI failures — the real UI rate + assert "Unit" in body and "Instrumented" in body and "UI automation" in body + assert "test_1_Foo" in body # failing test names still listed + assert out["record_as"] == "rc_report" and out["outbound"] is True + # no owner → blocked + st2 = ReleaseState(release_id="2026-08", ccd="2026-08-26") + out2 = as_dict(_steps.get_step("build_verify", "rc_report").build(st2)) + assert out2["kind"] == "blocked" and "owner" in out2["reason"] def test_rc_ui_gate_and_run_links(): @@ -2116,64 +2135,50 @@ def _model(ecs_ui, local_ui, ecs_suites=None): def test_record_rc_report_applies_ui_gate_and_stashes_links(): - """`record-rc-report` (the follow-up the skill runs after emailing) applies the 90% - UI gate: >=90% → step done; <90% → step BLOCKS (awaiting_action). Either way it - stashes the evaluated run links on the step. release_report is monkeypatched offline.""" + """`record-rc-report` (the follow-up the skill runs after emailing) reads the RC + snapshot from state and applies the 90% UI gate: >=90% → step done; <90% → step + BLOCKS (awaiting_action). Either way it stashes the evaluated run links on the step.""" import tempfile as _tf - from tools import pipelines as P from orchestrator.commands import rc_report as RR from orchestrator.state import StepState - def _model(ecs_ui, local_ui): - return {"release": "2026-08", - "checker": {"fired": True, "run_id": 111}, - "orchestrator": {"found": True, "run_id": 222}, - "mrwp": {"ECS": {"run_id": 333, "failed_suites": [], - "tests": {"categories": {"ui": ecs_ui}}}, - "Local": {"run_id": 444, "failed_suites": [], - "tests": {"categories": {"ui": local_ui}}}}, - "problems": []} - - orig = P.release_report with _tf.TemporaryDirectory() as d: rid = "2026-08" _stub_build_defs("pass") st = ReleaseState(release_id=rid, ccd="2026-08-26", owner_email="dev@microsoft.com") orch = Orchestrator(CONFIG, st) _pass_scout_checks(orch); orch.gate.sign() - C.save_state(st, d, rid) class A: runs_root = d; release = rid; config = CONFIG; as_of = None - try: - # PASS: 190/200 = 95% ≥ 90 → step done, links stashed - P.release_report = lambda *a, **k: _model( - {"total": 100, "passed": 95, "failed": 5}, - {"total": 100, "passed": 95, "failed": 5}) - assert RR.cmd_record_rc_report(A) == 0 - s1 = C.load_state(d, rid) - assert s1.is_done("build_verify", "rc_report") - step1 = s1.get_step("build_verify", "rc_report") - assert [l["name"] for l in step1.links] == [ - "Code Complete Checker run", "Release Orchestrator run", - "MRWP ECS run", "MRWP Local run"] - - # reset the step, then FAIL: 120/200 = 60% < 90 → blocked, links still stashed - s1.set_step("build_verify", "rc_report", StepState()) - C.save_state(s1, d, rid) - P.release_report = lambda *a, **k: _model( - {"total": 100, "passed": 60, "failed": 40}, - {"total": 100, "passed": 60, "failed": 40}) - assert RR.cmd_record_rc_report(A) == 2 - s2 = C.load_state(d, rid) - step2 = s2.get_step("build_verify", "rc_report") - assert step2.status == "blocked" and not s2.is_done("build_verify", "rc_report") - assert s2.status == "awaiting_action" - assert "build_verify.rc_report" in s2.pending_human - assert "BELOW" in step2.note and len(step2.links) == 4 - finally: - P.release_report = orig + # PASS: 190/200 = 95% ≥ 90 → step done, links stashed + _seed_rc_pipeline(st, {"total": 100, "passed": 95, "failed": 5}, + {"total": 100, "passed": 95, "failed": 5}) + C.save_state(st, d, rid) + assert RR.cmd_record_rc_report(A) == 0 + s1 = C.load_state(d, rid) + assert s1.is_done("build_verify", "rc_report") + step1 = s1.get_step("build_verify", "rc_report") + assert [l["name"] for l in step1.links] == [ + "Code Complete Checker run", "Release Orchestrator run", + "MRWP ECS run", "MRWP Local run"] + + # reset the step + re-seed the SAME runs with a failing UI slice (60% < 90) → + # blocked, links still stashed. Same run ids → updates the current rc in place. + s1.set_step("build_verify", "rc_report", StepState()) + _seed_rc_pipeline(s1, {"total": 100, "passed": 60, "failed": 40}, + {"total": 100, "passed": 60, "failed": 40}) + C.save_state(s1, d, rid) + assert RR.cmd_record_rc_report(A) == 2 + s2 = C.load_state(d, rid) + step2 = s2.get_step("build_verify", "rc_report") + assert step2.status == "blocked" and not s2.is_done("build_verify", "rc_report") + assert s2.status == "awaiting_action" + assert "build_verify.rc_report" in s2.pending_human + assert "BELOW" in step2.note and len(step2.links) == 4 + # the same rc was updated in place (not a spurious new RC iteration) + assert len(s2.pipeline_runs["rcs"]) == 1 def test_active_phase_report_steps_carry_links(): @@ -2334,22 +2339,65 @@ def test_mrwp_run_ids_picks_newest_on_retrigger(): def test_build_verify_persists_pipeline_run_ids(): - """The build_verify steps stash the resolved checker/orchestrator/MRWP run ids onto - state.pipeline_runs, and they round-trip through save/load (used by status + digest).""" + """The build_verify steps stash the checker/orchestrator/MRWP runs onto + state.pipeline_runs in the nested RC schema, and it round-trips through save/load.""" import tempfile from orchestrator import cli_common as _C st, orch = _bv_state({}) for sid in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): _bv_build(orch, st, sid) pr = st.pipeline_runs - assert pr.get("checker") == "1678599" - assert pr.get("orchestrator") == "1678611" - assert pr.get("mrwp_ecs") == "900001" and pr.get("mrwp_local") == "900002" - assert "Broker 1.0.0" in (pr.get("versions") or "") and pr.get("resolved_at") + assert pr["checker"]["run_id"] == "1678599" + assert pr["orchestrator"]["run_id"] == "1678611" + assert pr["orchestrator"]["versions"].get("Broker") == "1.0.0" + rc = pr["rcs"][-1] + assert rc["rc"] == 1 and rc.get("resolved_at") + assert rc["ecs"]["run_id"] == "900001" and rc["local"]["run_id"] == "900002" + assert rc["ecs"]["complete"] and rc["ecs"]["tests"]["failed"] == 4 # snapshot stored with tempfile.TemporaryDirectory() as tmp: _C.save_state(st, tmp, "2026-08") again = _C.load_state(tmp, "2026-08") - assert again.pipeline_runs.get("mrwp_ecs") == "900001" + assert again.pipeline_runs["rcs"][-1]["ecs"]["run_id"] == "900001" + + +def test_migrate_pipeline_runs_flat_to_nested(): + """A legacy FLAT pipeline_runs shape migrates to the nested RC schema on load + (idempotent); an already-nested value passes through unchanged.""" + from orchestrator.state import migrate_pipeline_runs as M + flat = {"checker": "111", "orchestrator": "222", + "versions": "Common 24.6.0, Msal 8.4.2, Broker 16.5.0", + "mrwp_ecs": "333", "mrwp_local": "444", "mrwp_id_source": "tags", + "resolved_at": "2026-08-20T00:00:00Z"} + m = M(flat) + assert m["checker"]["run_id"] == "111" + assert m["orchestrator"]["run_id"] == "222" + assert m["orchestrator"]["versions"] == {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"} + assert m["rcs"] == [{"rc": 1, "resolved_at": "2026-08-20T00:00:00Z", + "ecs": {"run_id": "333", "id_source": "tags"}, + "local": {"run_id": "444", "id_source": "tags"}}] + assert M(m) == m # idempotent + assert M({}) == {} + + +def test_stash_mrwp_appends_new_rc_on_id_change(): + """stash_mrwp merges ecs+local into ONE rc entry, and appends a NEW rc iteration only + when a provider's run id changes (RC Testing re-triggered). Latest = rcs[-1].""" + from steps.build_verify import _common as K + st = ReleaseState(release_id="2026-08") + ecs1 = {"run_id": "900001", "complete": True, "tests": {"categories": {"ui": {"total": 10, "passed": 9, "failed": 1}}}} + K.stash_mrwp(st, "ECS", ecs1) + K.stash_mrwp(st, "Local", {"run_id": "900002", "complete": True}) + assert len(st.pipeline_runs["rcs"]) == 1 # both merged into rc 1 + assert K.latest_rc(st)["rc"] == 1 + # re-resolving the SAME ecs id updates in place — no new rc + K.stash_mrwp(st, "ECS", ecs1) + assert len(st.pipeline_runs["rcs"]) == 1 + # a NEW ecs id → RC re-triggered → append rc 2 + K.stash_mrwp(st, "ECS", {"run_id": "910001", "complete": True}) + K.stash_mrwp(st, "Local", {"run_id": "910002", "complete": True}) + rcs = st.pipeline_runs["rcs"] + assert [r["rc"] for r in rcs] == [1, 2] + assert K.latest_rc(st)["ecs"]["run_id"] == "910001" # latest = last def test_digest_shows_rc_line_when_build_verify_active(): @@ -2361,9 +2409,10 @@ def test_digest_shows_rc_line_when_build_verify_active(): "show_pipeline_runs": True, "due": True, "started": True, "done": 2, "total": 5, "outstanding": [], "completed": ["checker_fired", "orchestrator_health"]}, - "pipeline_runs": {"checker": "1678599", "orchestrator": "1678611", - "versions": "Broker 1.0.0", "mrwp_ecs": "900001", - "mrwp_local": "900002"}} + "pipeline_runs": { + "checker": {"run_id": "1678599"}, + "orchestrator": {"run_id": "1678611", "versions": {"Broker": "1.0.0"}}, + "rcs": [{"rc": 1, "ecs": {"run_id": "900001"}, "local": {"run_id": "900002"}}]}} text = render.notification(r) md = render.notification_markdown(r) assert "RC pipelines:" in text and "orchestrator 1678611" in text @@ -2393,9 +2442,9 @@ def test_sim_fast_forwards_to_rc_gate_offline(): assert st.is_done("build_verify", s), s from orchestrator.engine import Orchestrator as _O assert _O(CONFIG, st).current_phase_id() == "bug_bash" # positioned past Phase 2 - # pipeline ids were stashed by the steps during the sim - assert st.pipeline_runs.get("orchestrator") == "1678611" - assert st.pipeline_runs.get("mrwp_ecs") == "1678863" + # pipeline runs were stashed by the steps during the sim (nested RC schema) + assert st.pipeline_runs["orchestrator"]["run_id"] == "1678611" + assert st.pipeline_runs["rcs"][-1]["ecs"]["run_id"] == "1678863" assert st.readiness_signed From aa0b2ba4fa135efd97486c34c06ef31b58f37a28 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 21:44:16 +0100 Subject: [PATCH 68/90] release-agent: apply the unit-test retry rule in the RC report (recovered = passed + warn) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit tests run under a retry rule: a flaky test can appear several times in one run (e.g. Passed/Failed/Passed) and ADO's run aggregate still counts it as a failure. Reconcile per-result by testCaseTitle so a test that passed on any attempt counts as PASSED; a failed-then-passed test is RECOVERED (counted passed, surfaced as a warning); only never-passed tests are real failures. Unit-only — UI/instrumented (and the UI gate) are unchanged. reconcile_retries() + a paged per-run results fetch; get_test_summary re-reads failing UNIT runs per-result; get_failed_tests drops recovered unit titles (and fully-recovered suites); the RC email (plain+HTML) and rc-report diagnostic show a retry warning listing recovered unit tests. Also fixes retry double-counting in totals. Verified live on build 1681651: testNullDrsMetadata + gated_notEnabled_logsNothingAtAll reconcile to passed, unit failed=0. 187/187 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../orchestrator/commands/rc_report.py | 11 ++ release-agent/steps/build_verify/_common.py | 36 +++++ release-agent/tests/test_engine.py | 85 ++++++++++++ release-agent/tools/pipelines.py | 126 ++++++++++++++++-- 4 files changed, 247 insertions(+), 11 deletions(-) diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 5e9b1908..9339985f 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -177,6 +177,17 @@ def _format(m) -> str: L += ["", "**Issues:**"] for p in probs: L.append(f" - {p}") + # Unit retry warning — failed-then-passed on retry (counted as passed). + recovered = sorted({t for prov in ("ECS", "Local") + for t in ((((m.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) + .get("categories", {}).get("unit", {}).get("recovered") or [])}) + if recovered: + L += ["", f"⚠ **Retry warning** — {len(recovered)} unit test(s) failed then passed " + f"on retry (counted as passed):"] + for t in recovered[:20]: + L.append(f" - {t}") + if len(recovered) > 20: + L.append(f" … and {len(recovered) - 20} more") return "\n".join(L) diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index ce28237c..56c98a1d 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -238,6 +238,19 @@ def rc_ui_gate(model) -> dict: f"to override." + _ui_failing_suites_summary(model))} +def recovered_unit_tests(model) -> list: + """Unit tests that FAILED then PASSED on retry (the unit retry rule) across both MRWP + providers — counted as passed, but surfaced as a warning in the report. De-duplicated, + sorted.""" + out = set() + for prov in ("ECS", "Local"): + cats = (((model.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) \ + .get("categories", {}) + for t in ((cats.get("unit") or {}).get("recovered") or []): + out.add(t) + return sorted(out) + + def rc_email_subject(model) -> str: rid = model.get("release", "?") v = rc_ui_gate(model)["verdict"] @@ -316,6 +329,14 @@ def _rc_email_plain(model, ctx) -> str: L.append("BLOCKING ISSUES (a stage that never ran = the pipeline aborted):") L += [f" - {p}" for p in probs] L.append("") + recovered = recovered_unit_tests(model) + if recovered: + L.append(f"\u26a0 RETRY WARNING — {len(recovered)} unit test(s) FAILED then PASSED " + f"on retry (counted as passed; verify they aren't genuinely flaky):") + L += [f" - {t}" for t in recovered[:20]] + if len(recovered) > 20: + L.append(f" … and {len(recovered) - 20} more") + L.append("") L.append("NEXT: review the failing tests above. If they're acceptable to carry into " "bug bash, approve the gate (advances to Phase 3 — Test / Bug Bash). " "Otherwise investigate the red suites first.") @@ -448,6 +469,20 @@ def _ui_sum(field): + "".join(f"

  • {T.esc(p)}
    • " for p in probs) + "
    ") if probs else "") + recovered = recovered_unit_tests(model) + retry_warn = "" + if recovered: + shown = recovered[:15] + more = (f"
  • … and " + f"{len(recovered) - len(shown)} more
    • " if len(recovered) > len(shown) else "") + retry_warn = ( + "
    ⚠ Retry warning — " + f"{len(recovered)} unit test(s) failed then passed on retry (counted as " + "passed; verify they aren’t genuinely flaky):" + "
      " + + "".join(f"
    • {T.esc(t)}
      • " for t in shown) + more + "
    ") + return f"""\
    @@ -484,6 +519,7 @@ def _ui_sum(field): {mrwp_card('ECS')} {mrwp_card('Local')} {issues} + {retry_warn}
    diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 2ee12a94..bb8fc3fa 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2189,6 +2189,91 @@ def test_active_phase_report_steps_carry_links(): assert ap and all("links" in s for s in ap["steps"]) +def test_reconcile_retries_pure(): + """reconcile_retries collapses per-attempt results by title: passed if any attempt + passed; recovered if it also failed; failed only if it never passed; NA ignored.""" + from tools import pipelines as P + res = [ + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Failed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, # flaky → recovered + {"testCaseTitle": "testAlwaysGreen", "outcome": "Passed"}, + {"testCaseTitle": "testHardFail", "outcome": "Failed"}, + {"testCaseTitle": "testHardFail", "outcome": "Failed"}, # failed every attempt + {"testCaseTitle": "testSkipped", "outcome": "NotExecuted"}, # ignored + ] + r = P.reconcile_retries(res) + assert r["passed"] == 2 and r["failed"] == 1 + assert r["recovered"] == ["testNullDrsMetadata"] + assert r["total"] == 3 and r["na"] == 1 + assert P.reconcile_retries([]) == {"passed": 0, "failed": 0, "recovered": [], "total": 0, "na": 0} + + +def test_get_test_summary_unit_retry_reconciles(): + """A UNIT run whose only failure is a flaky test that passed on retry reconciles to + 0 failed + a `recovered` warning — even though ADO's run aggregate said 1 failed. + (The rule is unit-only; UI/instrumented use the aggregate unchanged.)""" + from tools import pipelines as P + runs = {"value": [{"id": 700, "name": "broker4j_UnitTests", + "totalTests": 5, "passedTests": 4, "notApplicableTests": 0}]} + results = {"value": [ + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Failed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, + {"testCaseTitle": "t2", "outcome": "Passed"}, + {"testCaseTitle": "t3", "outcome": "Passed"}, + {"testCaseTitle": "t4", "outcome": "Passed"}, + ]} + orig = P._ado_rest_get + P._ado_rest_get = lambda url, timeout: (True, runs if "buildUri" in url else results, "") + try: + ok, s, _ = P.get_test_summary("O", "P", 700) + assert ok + unit = s["categories"]["unit"] + assert unit["failed"] == 0 and unit["recovered"] == ["testNullDrsMetadata"] + assert unit["passed"] == 4 and unit["total"] == 4 + finally: + P._ado_rest_get = orig + + +def test_get_failed_tests_drops_fully_recovered_unit_suite(): + """A unit suite whose only failure recovered on retry produces NO failing suite.""" + from tools import pipelines as P + runs = {"value": [{"id": 701, "name": "broker4j_UnitTests # 700_build.1", + "totalTests": 5, "passedTests": 4, "notApplicableTests": 0}]} + results = {"value": [ + {"testCaseTitle": "flaky", "outcome": "Failed"}, + {"testCaseTitle": "flaky", "outcome": "Passed"}, + ]} + orig = P._ado_rest_get + P._ado_rest_get = lambda url, timeout: (True, runs if "buildUri" in url else results, "") + try: + ok, suites, _ = P.get_failed_tests("O", "P", 701) + assert ok and suites == [] # the only failure recovered → no failing suite + finally: + P._ado_rest_get = orig + + +def test_rc_report_email_shows_retry_warning(): + """When a unit test recovered on retry, the RC report (plain + HTML) surfaces a retry + warning that lists it (counted as passed but flagged).""" + from steps.build_verify import _common as K + model = {"release": "2026-08", "checker": {"run_id": 1}, + "orchestrator": {"run_id": 2, "versions": {}, "parked": True}, + "mrwp": {"ECS": {"run_id": 3, "ran": 23, "total": 23, + "tests": {"categories": { + "unit": {"total": 100, "passed": 100, "failed": 0, + "recovered": ["testNullDrsMetadata"]}, + "ui": {"total": 10, "passed": 10, "failed": 0}}}}, + "Local": {"run_id": 4, "ran": 23, "total": 23, "tests": {"categories": {}}}}, + "problems": []} + assert K.recovered_unit_tests(model) == ["testNullDrsMetadata"] + plain = K._rc_email_plain(model, {}) + html = K._rc_email_html(model, {}) + assert "RETRY WARNING" in plain and "testNullDrsMetadata" in plain + assert "Retry warning" in html and "testNullDrsMetadata" in html + + def test_classify_test_run_categories(): """The test-run classifier buckets into exactly three: unit / instrumented / ui; anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 645f9510..961572eb 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -317,11 +317,73 @@ def classify_test_run(name): return "ui" +# Outcomes that are neither a pass nor a real failure (skipped / not run / inconclusive). +_NA_OUTCOMES = {"NotExecuted", "NotApplicable", "None", "Inconclusive", "Warning", None} + + +def reconcile_retries(results): + """Collapse ADO's per-attempt test results into ONE verdict per test (by title). + + UNIT tests run under a RETRY rule: a flaky test can appear several times in the same + run — e.g. Passed, Failed, Passed. ADO's run aggregate still counts that as a failure, + but the test ultimately PASSED. This groups results by testCaseTitle and rules: + * PASSED — at least one attempt Passed. + * RECOVERED — Passed AND Failed on different attempts (a flaky pass — surfaced as a + warning, but counted as passed). + * FAILED — has a real (non-NA) attempt and NEVER passed. + Not-executed / not-applicable attempts are ignored. Counts are DISTINCT tests. Returns + {passed, failed, recovered:[titles], total, na}.""" + import collections + by = collections.defaultdict(set) + for r in results or []: + title = (r.get("testCaseTitle") or r.get("automatedTestName") or "").strip() + if not title: + continue + by[title].add(r.get("outcome")) + passed = failed = na = 0 + recovered = [] + for title, outs in by.items(): + eff = {o for o in outs if o not in _NA_OUTCOMES} + if not eff: + na += 1 + continue + if "Passed" in eff: + passed += 1 + if "Failed" in eff: + recovered.append(title) + else: + failed += 1 + return {"passed": passed, "failed": failed, "recovered": sorted(recovered), + "total": passed + failed, "na": na} + + +def _run_results(org, project, run_id, timeout=90, page=1000, cap=10000): + """All test results for a run (paged). Returns (ok, [results], detail).""" + base = org.rstrip("/") + out, skip = [], 0 + while len(out) < cap: + url = (f"{base}/{project}/_apis/test/Runs/{run_id}/results" + f"?api-version=7.1&$top={page}&$skip={skip}") + ok, data, detail = _ado_rest_get(url, timeout) + if not ok: + return (False, out, detail) + batch = (data or {}).get("value", []) or [] + out.extend(batch) + if len(batch) < page: + break + skip += page + return (True, out, "") + + def get_test_summary(org, project, build_id, timeout=60): """Return (ok, summary, detail) for a build's Test-tab results. summary = - {total, passed, failed, runs:[{name,total,passed,failed,category}], - categories:{unit|instrumented|ui|other: {total,passed,failed}}} aggregated across - all test runs, classified into unit / instrumented / UI-automation / other. + {total, passed, failed, runs:[{name,total,passed,failed,category,recovered}], + categories:{unit|instrumented|ui: {total,passed,failed,recovered:[titles]}}} + aggregated across all test runs, classified into unit / instrumented / UI-automation. + + UNIT runs apply the retry rule (reconcile_retries): a run WITH failures is re-read at + the per-result level so a flaky test that failed-then-passed counts as passed (and is + reported as `recovered`). UI/instrumented runs use ADO's run aggregate unchanged. Uses the Test Runs REST API directly (az devops invoke mis-routes this one).""" base = org.rstrip("/") @@ -332,21 +394,31 @@ def get_test_summary(org, project, build_id, timeout=60): return (False, None, detail) runs = (data or {}).get("value", []) or [] out_runs, tot, passed = [], 0, 0 - cats = {c: {"total": 0, "passed": 0, "failed": 0} for c in TEST_CATEGORIES} + cats = {c: {"total": 0, "passed": 0, "failed": 0, "recovered": []} for c in TEST_CATEGORIES} for r in runs: t = r.get("totalTests") or 0 p = r.get("passedTests") or 0 na = r.get("notApplicableTests") or 0 f = max(t - p - na, 0) cat = classify_test_run(r.get("name")) + recovered = [] + # UNIT retry rule: re-read a failing unit run per-result and reconcile flaky passes. + if cat == "unit" and f > 0: + ok2, results, _ = _run_results(org, project, r.get("id"), timeout) + if ok2 and results: + rec = reconcile_retries(results) + t, p, f, recovered = rec["total"], rec["passed"], rec["failed"], rec["recovered"] tot += t passed += p cats[cat]["total"] += t cats[cat]["passed"] += p cats[cat]["failed"] += f + if recovered: + cats[cat]["recovered"].extend(recovered) out_runs.append({"name": r.get("name"), "total": t, "passed": p, - "failed": f, "category": cat}) - return (True, {"total": tot, "passed": passed, "failed": max(tot - passed, 0), + "failed": f, "category": cat, "recovered": recovered}) + failed_total = sum(c["failed"] for c in cats.values()) + return (True, {"total": tot, "passed": passed, "failed": failed_total, "runs": out_runs, "categories": cats}, "") @@ -359,9 +431,14 @@ def _suite_base_name(name): def get_failed_tests(org, project, build_id, max_result_calls=20, per_suite_cap=40, timeout=90): """Return (ok, suites, detail) — the individual FAILING tests for a build, aggregated by suite name (the same suite appears as multiple runs; merged). suites is a list of - {name, failed, total, tests:[test titles]}, sorted by failure count desc. Test titles - are fetched for the worst runs first, bounded by max_result_calls; per suite capped at - per_suite_cap names.""" + {name, failed, total, category, tests:[titles], recovered:[titles]}, sorted by failure + count desc. Test titles are fetched for the worst runs first, bounded by + max_result_calls; per suite capped at per_suite_cap names. + + UNIT suites apply the retry rule: a failing unit run is re-read per-result and + reconciled (reconcile_retries), so a flaky test that failed-then-passed is NOT listed + as a failure — it's collected under `recovered` and a suite that fully recovers is + dropped.""" base = org.rstrip("/") url = (f"{base}/{project}/_apis/test/runs" f"?buildUri=vstfs:///Build/Build/{build_id}&api-version=7.1") @@ -378,8 +455,33 @@ def fcount(r): suites, calls = {}, 0 for r in failing: name = _suite_base_name(r.get("name")) + cat = classify_test_run(name) s = suites.setdefault(name, {"name": name, "failed": 0, "total": 0, - "category": classify_test_run(name), "tests": []}) + "category": cat, "tests": [], "recovered": []}) + # UNIT retry rule: reconcile per-result so flaky-recovered tests aren't failures. + if cat == "unit": + ok2, results, _ = _run_results(org, project, r.get("id"), timeout) + if ok2: + rec = reconcile_retries(results) + s["failed"] += rec["failed"] + s["total"] += rec["total"] + for t in rec["recovered"]: + if t not in s["recovered"]: + s["recovered"].append(t) + # only the tests that truly failed (never passed) — reconcile again for names + import collections + by = collections.defaultdict(set) + for res in results: + title = (res.get("testCaseTitle") or res.get("automatedTestName") or "").strip() + if title: + by[title].add(res.get("outcome")) + for title, outs in by.items(): + eff = {o for o in outs if o not in _NA_OUTCOMES} + if eff and "Passed" not in eff and title not in s["tests"] \ + and len(s["tests"]) < per_suite_cap: + s["tests"].append(title) + continue + # NON-unit (UI / instrumented) — ADO aggregate + the failed titles (unchanged). s["failed"] += fcount(r) s["total"] += r.get("totalTests") or 0 if calls < max_result_calls: @@ -392,7 +494,9 @@ def fcount(r): title = (res.get("testCaseTitle") or res.get("automatedTestName") or "").strip() if title and title not in s["tests"] and len(s["tests"]) < per_suite_cap: s["tests"].append(title) - return (True, sorted(suites.values(), key=lambda x: -x["failed"]), "") + # Drop suites whose failures all recovered on retry (unit); keep real failures. + real = [s for s in suites.values() if s["failed"] > 0] + return (True, sorted(real, key=lambda x: -x["failed"]), "") # Coordinates for the release chain (identitydivision/Engineering). The build_verify From 82c2cc30020f3de5bbde2d65fa6c2845dd4dc715 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 22:03:33 +0100 Subject: [PATCH 69/90] =?UTF-8?q?release-agent:=20arch=20review=20batch=20?= =?UTF-8?q?1=20=E2=80=94=20single-source=20ADO=20coords,=20one=20RC-model?= =?UTF-8?q?=20builder,=20test=20net-guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H2 (single-source coordinates): the release toolchain spans MULTIPLE ADO orgs (identitydivision/Engineering for the verification chain, msazure/One for localization+CG, identitydivision/IdentityWiki for the wiki). tools/pipelines.py now names them explicitly (IDENTITYDIVISION/MSAZURE hosts, ENGINEERING_ORG/ENGINEERING_PROJECT) instead of a misleading global ORG/PROJECT; build_verify (Engineering-only) aliases them, cron/wiki import the right target, localization/cg keep their own. H1 (one RC-model builder): assemble_rc_model() is the single canonical builder; release_report (live) and rc_report_model (state) both route through it so they can't drift on shape or problem messages. Guard test asserts agreement; live-verified on 2026-08. L2 (test net-guard): tests/conftest.py autouse fixture makes any un-mocked ADO/az call raise loudly instead of hanging. 188 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/steps/build_verify/_common.py | 60 ++++---- release-agent/steps/preflight/cron.py | 7 +- release-agent/steps/preflight/wiki.py | 6 +- release-agent/tests/conftest.py | 38 +++++ release-agent/tests/test_engine.py | 26 ++++ release-agent/tools/pipelines.py | 145 +++++++++++++------- 6 files changed, 188 insertions(+), 94 deletions(-) create mode 100644 release-agent/tests/conftest.py diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 56c98a1d..7dcec4a8 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -1,26 +1,21 @@ """Shared config + helpers for the Phase 2 (build_verify) release-verification steps. -Underscore-prefixed so steps.discover() skips it (it's not a step). Holds the ADO -coordinates for the three Engineering release pipelines and the recovery / escalation -links surfaced when a step blocks, plus small resolvers the step modules reuse. +Underscore-prefixed so steps.discover() skips it (it's not a step). This whole package is +scoped to ONE ADO target — identitydivision/Engineering (the release-verification chain) — +so the local `ORG`/`PROJECT` names here are unambiguous. Their VALUES are imported from +`tools.pipelines` (the single source; other areas like localization use a DIFFERENT org), +so a coordinate change happens in exactly one place. This module adds the recovery / +escalation links surfaced when a step blocks, plus small resolvers the step modules reuse. """ from __future__ import annotations -ORG = "https://identitydivision.visualstudio.com" -PROJECT = "Engineering" - -CHECKER_DEF = 3038 # Code Complete Calendar Checker (fires the release on the CCD) -ORCHESTRATOR_DEF = 2828 # Release Orchestrator (the spine) -MRWP_DEF = 2519 # Monthly Release Work Pipeline (RC testing; runs ECS + Local) - -# The orchestrator stages that must be green before RC testing is trustworthy, and the -# stage it should be PARKED at (a human approval gate the owner clears in a later phase). -ORCH_REQUIRED_STAGES = [ - "Validate Branch and Versions availability", - "Create Release Branches", - "Trigger RC Testing", -] -ORCH_PARK_STAGE = "Remove RC Tags" +# This package is Engineering-only; alias the explicitly-named source constants to the +# short local names the step CONFIGs use. (Do NOT use these for msazure/One calls.) +from tools.pipelines import ( + ENGINEERING_ORG as ORG, ENGINEERING_PROJECT as PROJECT, + CHECKER_DEF, ORCHESTRATOR_DEF, MRWP_DEF, + ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, +) # Surfaced in every block reason so the engineer knows how to recover / escalate. RECOVERY_TSG = ("https://eng.ms/docs/microsoft-security/identity/" @@ -116,11 +111,10 @@ def stash_mrwp(state, provider, snapshot): def rc_report_model(state, timeout=120): """The Phase-2 RC report model — assembled from the RECORD in state.pipeline_runs (the verification steps stored it), NOT a live re-discovery. Uses the LATEST RC - iteration (rcs[-1]). Shape mirrors tools.pipelines.release_report so the gate + email - builders consume it unchanged: - {release, checker{fired,run_id,when}, orchestrator{found,healthy,parked,run_id,versions}, - mrwp{ECS{...}, Local{...}}, problems[], rc} + iteration (rcs[-1]) and routes through `tools.pipelines.assemble_rc_model` — the SAME + assembler the live path uses — so the state-based model can't drift from the live one. """ + from tools import pipelines as P from orchestrator.state import migrate_pipeline_runs pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) ch = pr.get("checker") or {} @@ -128,20 +122,16 @@ def rc_report_model(state, timeout=120): rcs = pr.get("rcs") or [] rc = rcs[-1] if rcs else {} - model = { - "release": state.release_id, - "checker": {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), - "when": ch.get("when")}, - "orchestrator": {"found": bool(o.get("run_id")), "healthy": True, - "run_id": o.get("run_id"), "versions": o.get("versions") or {}, - "parked": o.get("parked")}, - "mrwp": {}, "problems": [], "rc": rc.get("rc"), - } + checker = {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), "when": ch.get("when")} + orchestrator = {"found": bool(o.get("run_id")), "healthy": True, + "run_id": o.get("run_id"), "versions": o.get("versions") or {}, + "parked": o.get("parked")} + mrwp = {} for slot, prov in (("ecs", "ECS"), ("local", "Local")): s = rc.get(slot) if not s: continue - model["mrwp"][prov] = { + mrwp[prov] = { "run_id": s.get("run_id"), "complete": s.get("complete"), "ran": s.get("ran"), "total": s.get("total"), "failed_stages": s.get("failed_stages") or [], @@ -149,11 +139,7 @@ def rc_report_model(state, timeout=120): "never_ran": s.get("never_ran") or [], "tests": s.get("tests"), "failed_suites": s.get("failed_suites"), } - if not s.get("complete") and s.get("never_ran"): - model["problems"].append( - f"MRWP {prov}: did NOT run to completion — never-ran: " - f"{', '.join(n for n in s['never_ran'] if n)}.") - return model + return P.assemble_rc_model(state.release_id, checker, orchestrator, mrwp, rc=rc.get("rc")) def rc_run_links(model) -> list: diff --git a/release-agent/steps/preflight/cron.py b/release-agent/steps/preflight/cron.py index edd482c2..a3ace5a8 100644 --- a/release-agent/steps/preflight/cron.py +++ b/release-agent/steps/preflight/cron.py @@ -10,16 +10,17 @@ from orchestrator.outcomes import Done, Blocked from steps.lib.agent import legacy_run from steps.lib.mockctx import mock_input, MISSING +from tools.pipelines import ENGINEERING_ORG, ENGINEERING_PROJECT ID = "cron" KIND = "agent" # Step config (co-located). Pipeline 3038's cron proves it's FIRING via a recent -# schedule-reason run in its build history. +# schedule-reason run in its build history. It's the Engineering Calendar Checker. CONFIG = { "pipeline_id": 3038, - "org": "https://identitydivision.visualstudio.com", - "project": "Engineering", + "org": ENGINEERING_ORG, + "project": ENGINEERING_PROJECT, "name": "Code Complete Calendar Checker", "max_staleness_days": 2, # a daily cron should never be older than this } diff --git a/release-agent/steps/preflight/wiki.py b/release-agent/steps/preflight/wiki.py index cca38f16..4648d0cb 100644 --- a/release-agent/steps/preflight/wiki.py +++ b/release-agent/steps/preflight/wiki.py @@ -10,14 +10,16 @@ from orchestrator.outcomes import Done, Blocked from steps.lib.agent import legacy_run from steps.lib.mockctx import mock_input +from tools.pipelines import IDENTITYDIVISION ID = "wiki" KIND = "agent" # Step config (co-located). Creates the per-release payload page under the standing -# history parent page (Phase 2 later writes the built versions into it). +# history parent page (Phase 2 later writes the built versions into it). NOTE: this is a +# DIFFERENT project (IdentityWiki) from the release chain — only the org host is shared. CONFIG = { - "org": "https://identitydivision.visualstudio.com", + "org": IDENTITYDIVISION, # same collection host as Engineering, different project "project": "IdentityWiki", "wiki": "IdentityWiki.wiki", "parent_path": "/IdentityWiki/Services/Microsoft Authenticator/Release/Android/Monthly Releases Payloads History", diff --git a/release-agent/tests/conftest.py b/release-agent/tests/conftest.py new file mode 100644 index 00000000..b741df60 --- /dev/null +++ b/release-agent/tests/conftest.py @@ -0,0 +1,38 @@ +"""Shared pytest setup for the Release Orchestrator tests. + +Adds the package root to sys.path (tests import `orchestrator.*` / `tools.*` / `steps.*`) +and installs an AUTOUSE network guard: the real ADO/az primitives in `tools.pipelines` +are replaced with a raiser, so any test that reaches a live network call FAILS LOUDLY +with a clear message instead of hanging on `az`. Tests that need controlled responses +monkeypatch these primitives themselves (e.g. `P._ado_rest_get = fake`), which overrides +the guard for the duration of that test. +""" +from __future__ import annotations + +import os +import sys + +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # release-agent/ +if ROOT not in sys.path: + sys.path.insert(0, ROOT) + + +@pytest.fixture(autouse=True) +def _no_real_network(monkeypatch): + """Block real ADO/az calls in tests. Any un-mocked network access raises with a hint + naming what to patch — this is what turns an accidental live call into a fast, clear + failure instead of a multi-minute hang.""" + from tools import pipelines as P + + def _blocked(*_a, **_k): + raise RuntimeError( + "test attempted a REAL ADO/az network call — mock it " + "(patch tools.pipelines._ado_rest_get / _ado_rest_get_text / _az_json, " + "or inject the step's input mocks).") + + monkeypatch.setattr(P, "_ado_rest_get", _blocked) + monkeypatch.setattr(P, "_ado_rest_get_text", _blocked) + monkeypatch.setattr(P, "_az_json", _blocked) + yield diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index bb8fc3fa..41add017 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2274,6 +2274,32 @@ def test_rc_report_email_shows_retry_warning(): assert "Retry warning" in html and "testNullDrsMetadata" in html +def test_rc_model_shape_agrees_across_live_and_state_paths(): + """H1 guard: the live builder (pipelines.release_report → assemble_rc_model) and the + state builder (steps._common.rc_report_model → assemble_rc_model) produce the SAME + top-level model shape, so the gate/email/diagnostic never drift.""" + from tools import pipelines as P + from steps.build_verify import _common as K + # a canonical assembled model has exactly these top-level keys + m = P.assemble_rc_model("2026-08", {"fired": True, "run_id": 1}, + {"found": True, "healthy": True, "run_id": 2, "versions": {}}, + {"ECS": {"run_id": 3, "complete": True}}, rc=1, id_source="tags") + assert set(m) == {"release", "checker", "orchestrator", "mrwp", "problems", "rc", "mrwp_id_source"} + # the state path yields the same core keys (no id_source — that's live-only) + st = ReleaseState(release_id="2026-08") + _seed_rc_pipeline(st, {"total": 10, "passed": 10, "failed": 0}, + {"total": 10, "passed": 10, "failed": 0}) + sm = K.rc_report_model(st) + assert {"release", "checker", "orchestrator", "mrwp", "problems", "rc"} <= set(sm) + assert sm["rc"] == 1 and sm["problems"] == [] + # a never-ran MRWP snapshot yields the SAME problem string the live path derives + st2 = ReleaseState(release_id="2026-08") + from steps.build_verify import _common as K2 + K2.stash_mrwp(st2, "ECS", {"run_id": "9", "complete": False, "never_ran": ["UI Automation"]}) + pm = K2.rc_report_model(st2) + assert any("did NOT run to completion" in p and "UI Automation" in p for p in pm["problems"]) + + def test_classify_test_run_categories(): """The test-run classifier buckets into exactly three: unit / instrumented / ui; anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 961572eb..9df8b9a5 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -499,12 +499,25 @@ def fcount(r): return (True, sorted(real, key=lambda x: -x["failed"]), "") -# Coordinates for the release chain (identitydivision/Engineering). The build_verify -# steps carry their own copies in steps/build_verify/_common.py; this default lets the -# report aggregator run standalone. -CHECKER_DEF = 3038 -ORCHESTRATOR_DEF = 2828 -MRWP_DEF = 2519 +# ── ADO targets — the release toolchain spans MULTIPLE orgs/projects ───────────────── +# There is NO single global org/project. Name each target explicitly so a caller can't +# accidentally point a call at the wrong one: +# • identitydivision — the collection hosting Engineering (release chain) + IdentityWiki +# • msazure — hosts One (localization pipeline 405133, Component Governance) +# These constants cover ONLY the release-VERIFICATION chain (checker / orchestrator / MRWP), +# which lives in identitydivision/Engineering. Other areas own their own coordinates: +# localization → steps/ccd/localization.py (msazure/One); wiki → steps/preflight/wiki.py +# (identitydivision/IdentityWiki); CG → steps/preflight/cg.py (msazure/One). +IDENTITYDIVISION = "https://identitydivision.visualstudio.com" +MSAZURE = "https://msazure.visualstudio.com" + +# The release-verification pipelines: identitydivision / Engineering — SINGLE SOURCE. +# `steps/build_verify/_common.py` imports these (it does not redefine them). +ENGINEERING_ORG = IDENTITYDIVISION +ENGINEERING_PROJECT = "Engineering" +CHECKER_DEF = 3038 # Code Complete Calendar Checker (fires the release on the CCD) +ORCHESTRATOR_DEF = 2828 # Release Orchestrator (the spine) +MRWP_DEF = 2519 # Monthly Release Work Pipeline (RC testing; runs ECS + Local) TRIGGER_JOB = "Trigger Monthly Release" ORCH_REQUIRED_STAGES = [ "Validate Branch and Versions availability", @@ -514,33 +527,79 @@ def fcount(r): ORCH_PARK_STAGE = "Remove RC Tags" +def assemble_rc_model(release, checker, orchestrator, mrwp, *, rc=None, + id_source=None, io_problems=None): + """The ONE canonical Phase-2 RC report model — built from already-resolved pieces, + whether they came from LIVE reads (release_report) or the state snapshot + (steps.build_verify._common.rc_report_model). Both paths call this so they can never + drift on shape or on how `problems` are derived. + + Sections: + checker : {fired, run_id, when} | {fired:False} | {error} + orchestrator : {found, healthy, run_id, versions, parked, failed_stages, park_stage} | {found:False[,error]} + mrwp : {ECS:{...}, Local:{...}} each: run_id, complete, ran, total, + failed_stages, yellow_stages, never_ran, tests, failed_suites [, error] + `problems` is derived here in section order (checker → orchestrator → io_problems → + mrwp) from each section's `error`/structural state, so the messages are identical for + both callers. `io_problems` are non-sectional read failures (e.g. MRWP id resolution). + Returns {release, checker, orchestrator, mrwp, problems, rc[, mrwp_id_source]}. + """ + checker = checker or {} + orchestrator = orchestrator or {} + mrwp = mrwp or {} + problems = [] + + if checker.get("error"): + problems.append(f"Checker: could not read runs ({checker['error']}).") + elif checker.get("fired") is False: + problems.append("Checker: no run has a succeeded 'Trigger Monthly Release' job " + "(release not triggered yet, or before Code Complete Day).") + + o = orchestrator + if o.get("found") is False: + problems.append(f"Orchestrator: could not read runs ({o['error']})." if o.get("error") + else f"Orchestrator: no run found for {release}.") + elif o.get("error"): + problems.append(f"Orchestrator: could not read stages ({o['error']}).") + elif o.get("failed_stages"): + problems.append("Orchestrator: pre-gate stage(s) not green: " + + ", ".join(o["failed_stages"]) + ".") + + problems.extend(io_problems or []) + + for provider in ("ECS", "Local"): + e = mrwp.get(provider) + if not e: + continue + if e.get("error"): + problems.append(f"MRWP {provider}: could not read stages ({e['error']}).") + elif e.get("complete") is False: + nv = ", ".join(n for n in (e.get("never_ran") or []) if n) or "(unknown)" + problems.append(f"MRWP {provider}: did NOT run to completion — never-ran: {nv}.") + + model = {"release": release, "checker": checker, "orchestrator": orchestrator, + "mrwp": mrwp, "problems": problems, "rc": rc} + if id_source is not None: + model["mrwp_id_source"] = id_source + return model + + def release_report(org, project, release_month, checker_def=CHECKER_DEF, orch_def=ORCHESTRATOR_DEF, timeout=90, with_failed_tests=True): - """Assemble the full Phase-2 RC-pipeline + test report for a release month — a pure - read aggregator over the other helpers (does NOT gate; it reports). Returns a model: - - {release, checker:{fired,run_id,when}, - orchestrator:{run_id, found, healthy, parked, park_stage, failed_stages, versions}, - mrwp:{ECS:{...}, Local:{...}}, # each: run_id, complete, ran, total, failed_stages, - # yellow_stages, never_ran, tests:{total,passed,failed,runs}, - # failed_suites:[{name,failed,total,tests:[titles]}] - problems:[...]} # human-readable issues (empty = all good) + """Assemble the full Phase-2 RC-pipeline + test report for a release month by LIVE + reads (does NOT gate; it reports). Resolves the checker / orchestrator / both MRWP + runs, then hands the pieces to `assemble_rc_model` (the shared assembler) so this live + path and the state-snapshot path produce an identical model shape + problems. Any + field that couldn't be read carries an `error` note (surfaced as a problem). `with_failed_tests` (default True) also fetches the individual failing test names per - suite (extra REST calls — set False for a faster stage-only view). Any field that - couldn't be read carries an `error` note; those also land in `problems`. - """ - model = {"release": release_month, "checker": {}, "orchestrator": {}, - "mrwp": {}, "problems": []} - P = problems = model["problems"] - + suite (extra REST calls — set False for a faster stage-only view).""" # --- checker (did the release fire?) --- ok, runs, detail = find_checker_runs(org, project, checker_def, release_month, timeout) - fired = None if not ok: - model["checker"] = {"error": detail} - problems.append(f"Checker: could not read runs ({detail}).") + checker = {"error": detail} else: + fired = None for run in (runs or [])[:25]: ok2, recs, _ = get_timeline(org, project, run.get("id"), timeout) if not ok2: @@ -549,24 +608,15 @@ def release_report(org, project, release_month, checker_def=CHECKER_DEF, if rec is not None and rec.get("result") == "succeeded": fired = run break - if fired: - model["checker"] = {"fired": True, "run_id": fired.get("id"), - "when": (fired.get("queueTime") or "")[:16]} - else: - model["checker"] = {"fired": False} - problems.append("Checker: no run has a succeeded 'Trigger Monthly Release' job " - "(release not triggered yet, or before Code Complete Day).") + checker = ({"fired": True, "run_id": fired.get("id"), + "when": (fired.get("queueTime") or "")[:16]} if fired else {"fired": False}) # --- orchestrator (healthy? parked?) --- ok, orun, detail = find_orchestrator_run(org, project, orch_def, release_month, timeout) if not ok: - model["orchestrator"] = {"found": False, "error": detail} - problems.append(f"Orchestrator: could not read runs ({detail}).") - return model # can't resolve MRWP without the orchestrator + return assemble_rc_model(release_month, checker, {"found": False, "error": detail}, {}) if not orun: - model["orchestrator"] = {"found": False} - problems.append(f"Orchestrator: no run found for {release_month}.") - return model + return assemble_rc_model(release_month, checker, {"found": False}, {}) oid, tags = orun.get("id"), (orun.get("tags") or []) versions = {k: _tag_value(tags, f"Next{k}Version") for k in ("Common", "Msal", "Broker")} @@ -575,7 +625,6 @@ def release_report(org, project, release_month, checker_def=CHECKER_DEF, "park_stage": ORCH_PARK_STAGE, "failed_stages": [], "healthy": None, "parked": None} if not ok: o["error"] = detail - problems.append(f"Orchestrator: could not read stages ({detail}).") else: by = {s.get("name"): s for s in ostages} o["failed_stages"] = [n for n in ORCH_REQUIRED_STAGES @@ -583,37 +632,29 @@ def release_report(org, project, release_month, checker_def=CHECKER_DEF, o["healthy"] = not o["failed_stages"] park = by.get(ORCH_PARK_STAGE) o["parked"] = bool(park and park.get("state") != "completed") - if o["failed_stages"]: - problems.append("Orchestrator: pre-gate stage(s) not green: " - + ", ".join(o["failed_stages"]) + ".") - model["orchestrator"] = o # --- the two MRWP runs (ran to completion? tests?) --- ok, ids, detail, source = mrwp_run_ids(org, project, orun, timeout) if not ok: - problems.append(f"MRWP: could not resolve run ids ({detail}).") - return model - model["mrwp_id_source"] = source # 'tags' or 'logs' + return assemble_rc_model(release_month, checker, o, {}, + io_problems=[f"MRWP: could not resolve run ids ({detail})."]) + mrwp = {} for provider in ("ECS", "Local"): bid = ids.get(provider) entry = {"run_id": bid} ok, stages, detail = get_stages(org, project, bid, timeout) if not ok: entry["error"] = detail - problems.append(f"MRWP {provider}: could not read stages ({detail}).") else: comp = stage_completion(stages) entry.update({"complete": comp["complete"], "ran": comp["ran"], "total": comp["total"], "failed_stages": comp["failed"], "yellow_stages": comp["yellow"], "never_ran": comp["never_ran"]}) - if not comp["complete"]: - nv = ", ".join(n for n in comp["never_ran"] if n) or "(unknown)" - problems.append(f"MRWP {provider}: did NOT run to completion — never-ran: {nv}.") okt, tests, _ = get_test_summary(org, project, bid, timeout) entry["tests"] = tests if okt else None # Individual failing tests, aggregated by suite (deduped across repeated runs). if with_failed_tests and bid and tests and tests.get("failed"): okf, suites, _ = get_failed_tests(org, project, bid, timeout=timeout) entry["failed_suites"] = suites if okf else None - model["mrwp"][provider] = entry - return model + mrwp[provider] = entry + return assemble_rc_model(release_month, checker, o, mrwp, id_source=source) From 01cad029206c24a7d4d1a68dd8553cb3990552ec Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 22:15:49 +0100 Subject: [PATCH 70/90] =?UTF-8?q?release-agent:=20arch=20review=20batch=20?= =?UTF-8?q?2=20=E2=80=94=20extract=20status=20views,=20dedupe=20renderers,?= =?UTF-8?q?=20tidy=20imports/excepts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1: extract the ~250-line status view-model (_active_phase_report/_phase_map/_current_steps/_hold_view/_scheduled_view/status_report) into orchestrator/status_views.py as StatusViewMixin; engine.py 783->540 lines. Behaviour identical. M2: sort_failed_suites() + shared recovered_unit_tests() reused across the three RC renderers (plain email / HTML email / CLI report) — one place to change ordering. M3: format_versions() helper collapses six duplicated 'Common/Msal/Broker' join idioms. M4: hoisted the safe lazy imports in build_verify/_common (tools.pipelines, state, datetime, outcomes, mockctx) — no import cycle; left the automations/steps lazies that guard real cycles. L1: narrowed parse-y broad excepts (discovery JSON, engine/schedule YAML+zoneinfo, cli stream reconfigure) to specific exception types; left the intentional best-effort IO ones. L3: documented the orchestrator healthy=True invariant in rc_report_model. 188 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/cli.py | 2 +- .../orchestrator/commands/rc_report.py | 10 +- release-agent/orchestrator/discovery.py | 2 +- release-agent/orchestrator/engine.py | 260 +---------------- release-agent/orchestrator/render.py | 4 +- release-agent/orchestrator/schedule.py | 2 +- release-agent/orchestrator/status_views.py | 264 ++++++++++++++++++ release-agent/steps/build_verify/_common.py | 45 +-- .../steps/build_verify/orchestrator_health.py | 2 +- release-agent/tools/pipelines.py | 10 + 10 files changed, 314 insertions(+), 287 deletions(-) create mode 100644 release-agent/orchestrator/status_views.py diff --git a/release-agent/orchestrator/cli.py b/release-agent/orchestrator/cli.py index d39e49a6..2e2e8211 100644 --- a/release-agent/orchestrator/cli.py +++ b/release-agent/orchestrator/cli.py @@ -18,7 +18,7 @@ try: sys.stdout.reconfigure(encoding="utf-8") sys.stderr.reconfigure(encoding="utf-8") -except Exception: +except (AttributeError, ValueError): # non-reconfigurable stream / unsupported encoding pass HERE = os.path.dirname(os.path.abspath(__file__)) diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 9339985f..54a7163b 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -115,8 +115,7 @@ def _format(m) -> str: err = f" ({o['error']})" if "error" in o else "" L.append(f"⛔ **Release Orchestrator** — no run found{err}.") else: - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or "versions n/a" + vstr = K.format_versions(o.get("versions"), fallback="versions n/a") if o.get("healthy"): park = "parked at 'Remove RC Tags' (awaiting owner approval)" if o.get("parked") \ else f"'{o.get('park_stage')}' already cleared" @@ -158,8 +157,7 @@ def _format(m) -> str: # Failing tests, grouped by suite (UI first), each tagged by category. suites = r.get("failed_suites") if suites: - _ord = {"ui": 0, "instrumented": 1, "unit": 2} - for s in sorted(suites, key=lambda s: (_ord.get(s.get("category", "ui"), 9), -s["failed"])): + for s in K.sort_failed_suites(suites): cat = _lbl.get(s.get("category", "ui"), "UI automation") fr = round(s["failed"] * 100.0 / s["total"], 1) if s["total"] else 0.0 L.append(f" • [{cat}] {s['name']}: {s['failed']}/{s['total']} failed ({fr}%)") @@ -178,9 +176,7 @@ def _format(m) -> str: for p in probs: L.append(f" - {p}") # Unit retry warning — failed-then-passed on retry (counted as passed). - recovered = sorted({t for prov in ("ECS", "Local") - for t in ((((m.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) - .get("categories", {}).get("unit", {}).get("recovered") or [])}) + recovered = K.recovered_unit_tests(m) if recovered: L += ["", f"⚠ **Retry warning** — {len(recovered)} unit test(s) failed then passed " f"on retry (counted as passed):"] diff --git a/release-agent/orchestrator/discovery.py b/release-agent/orchestrator/discovery.py index b21c13a1..6e8cd056 100644 --- a/release-agent/orchestrator/discovery.py +++ b/release-agent/orchestrator/discovery.py @@ -18,7 +18,7 @@ def _summarize(state_file: str) -> Optional[dict]: try: with open(state_file, "r", encoding="utf-8") as fh: data = json.load(fh) - except Exception: + except (OSError, ValueError): # missing/unreadable file or bad JSON → skip it return None return { "release_id": data.get("release_id"), diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 868f53d5..ae4ac52d 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -21,6 +21,7 @@ from .readiness import ReadinessGate from . import schedule from . import mocks as mocks_mod +from .status_views import StatusViewMixin from steps.lib import mockctx import steps from phases import stub_runner @@ -36,7 +37,7 @@ class NextAction: message: str = "" -class Orchestrator: +class Orchestrator(StatusViewMixin): """The conductor: owns the state machine, dispatch loop, gates, and structured status. The readiness entry gate is delegated to ReadinessGate (self.gate); presentation lives in render.py. This class holds no formatting logic.""" @@ -89,7 +90,7 @@ def _config_timezone(config_path: str) -> Optional[str]: if os.path.exists(p): with open(p, "r", encoding="utf-8") as fh: return (yaml.safe_load(fh) or {}).get("timezone") - except Exception: + except (OSError, yaml.YAMLError): # missing/unreadable/invalid schedule.yaml pass return None @@ -591,262 +592,11 @@ def deny_gate(self, comment: str = "") -> NextAction: return NextAction(kind="gate", phase=phase, step=step, message=f"Gate DENIED: {phase} → {step}. Release blocked. {comment}".strip()) - # ---- reporting ---- + # `_phase_included` is a shared helper (used by both the state machine and the status + # views mixin). The status view-model builders live in orchestrator/status_views.py. def _phase_included(self, phase: dict) -> bool: return (not phase.get("conditional")) or phase["id"] in self._activated_conditionals() - def _active_phase_report(self) -> Optional[dict]: - """The first incomplete included phase, with its outstanding steps and - whether its time-window is open (due). This is what the daily phase - notification reports on — independent of state.current_phase (which is - only set once the release has been advanced).""" - for phase in self.config["phases"]: - if not self._phase_included(phase): - continue - steps = phase["steps"] - done = sum(1 for s in steps if self.state.is_done(phase["id"], s["id"])) - if done == len(steps): - continue # phase complete — look at the next one - outstanding = [ - {"id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), - "reminder": self._is_reminder(s), "owner": s.get("owner", "agent")} - for s in steps if not self.state.is_done(phase["id"], s["id"]) - ] - completed = [s["name"] for s in steps - if self.state.is_done(phase["id"], s["id"])] - cur = self.state.current_step - steps_view = [] - for s in steps: - sid = s["id"] - stp = self.state.get_step(phase["id"], sid) - s_done = self.state.is_done(phase["id"], sid) - s_blocked = stp.status == "blocked" - is_gate = bool(s.get("gate")) - is_rem = self._is_reminder(s) - is_scout = s.get("source") == "scout" - is_attest = bool(s.get("attest")) - if s_done: - status = "done" - elif s_blocked: - status = "blocked" - elif is_gate: - status = "approval" - elif is_attest: - status = "confirm" - elif is_rem: - status = "action" # a human to-do — the user must act - elif is_scout: - status = "scout" # Scout runs it automatically (scrape/send via MCP) - else: - status = "auto" - # needs_owner = a genuine USER task. A pending scout step is Scout's - # automatic work (not the user's) until it BLOCKS (s_blocked), so it is - # NOT flagged — only gates, reminders, attests, and blocks are. - needs = bool((is_gate or is_rem or is_attest or s_blocked) and not s_done) - steps_view.append({ - "id": sid, "name": s["name"], "status": status, - "needs_owner": needs, - "time_ready": self._step_time_ready(phase, s), # False = waits for its fire_at_local - "note": stp.note, # agent result / block reason / detail - "links": list(getattr(stp, "links", None) or []), # durable refs to items evaluated - "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_attest or s_blocked)), - }) - opens = self._phase_anchor_date(phase) - return { - "id": phase["id"], "name": phase["name"], - "num": phase.get("checklist_phase"), - "show_pipeline_runs": bool(phase.get("show_pipeline_runs")), - "done": done, "total": len(steps), - "due": self._phase_due(phase), "started": done > 0, - "opens": opens.isoformat() if opens else None, - "opens_in_days": (opens - self.as_of).days if opens else None, - "outstanding": outstanding, - "completed": completed, - "steps": steps_view, - } - return None - - def _phase_map(self): - """Build the phase overview + running totals. Returns - (phases, total, done, current_phase_name, current_phase_obj, current_step_name).""" - phases = [] - total = done = 0 - current_phase_name = current_step_name = None - current_phase_obj = None - for idx, phase in enumerate(self.config["phases"]): - if not self._phase_included(phase): - continue - p_total = len(phase["steps"]) - p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) - total += p_total - done += p_done - is_current = self.state.current_phase == phase["id"] - due = self._phase_due(phase) - opens = self._phase_anchor_date(phase) - if p_total and p_done == p_total: - state = "done" - elif not due and p_done == 0: - state = "scheduled" - elif is_current or p_done > 0: - state = "current" - else: - state = "pending" - if is_current: - current_phase_name = phase["name"] - current_phase_obj = phase - phases.append({ - "id": phase["id"], "name": phase["name"], - "num": phase.get("checklist_phase", idx), - "done": p_done, "total": p_total, "state": state, - "current": is_current, - "anchor": phase.get("anchor"), - "opens": opens.isoformat() if opens else None, - "opens_in_days": (opens - self.as_of).days if opens else None, - }) - for s in phase["steps"]: - if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: - current_step_name = s["name"] - return phases, total, done, current_phase_name, current_phase_obj, current_step_name - - def _current_steps(self, current_phase_obj) -> list: - """The current phase's steps, each tagged with a display state.""" - if not current_phase_obj: - return [] - phase_due = self._phase_due(current_phase_obj) - out = [] - for s in current_phase_obj["steps"]: - rec = self.state.steps.get(self.state.key(current_phase_obj["id"], s["id"]), {}) or {} - is_scout = s.get("source") == "scout" and not s.get("attest") - if rec.get("status") == "skipped": - s_state = "skipped" - elif self.state.is_done(current_phase_obj["id"], s["id"]): - s_state = "done" - elif rec.get("status") == "blocked": - s_state = "blocked" # a step hit a real problem — needs the owner - elif s["id"] == self.state.current_step and self.state.status == "holding_gate": - s_state = "gate" - elif is_scout: - s_state = "scout" # Scout's automatic work — never a user "do this" - elif s["id"] == self.state.current_step and self.state.status == "awaiting_action": - s_state = "reminder" - elif not phase_due: - s_state = "scheduled" - else: - s_state = "pending" - out.append({ - "id": s["id"], "name": s["name"], - "gate": bool(s.get("gate")), - "reminder": self._is_reminder(s), - "owner": s.get("owner", "agent"), - "state": s_state, - "note": rec.get("note"), # agent result / block reason / detail - "links": rec.get("links") or [], # durable refs (wiki page, CG alerts) - }) - return out - - def _hold_view(self, phase_name, step_name) -> dict: - """Detail of the current hold (gate or action-needed) — same shape for both.""" - return { - "phase": self.state.current_phase, - "phase_name": phase_name, - "step": self.state.current_step, - "step_name": step_name, - } - - def _scheduled_view(self) -> Optional[dict]: - """The phase we're waiting on the clock for — derived from the first - incomplete phase's due-ness, so `status` shows it even before `next`.""" - first_incomplete = next( - (p for p in self.config["phases"] - if self._phase_included(p) - and not all(self.state.is_done(p["id"], s["id"]) for s in p["steps"])), - None) - if (first_incomplete is None or self._phase_due(first_incomplete) - or self.state.status in ("complete", "halted", "blocked")): - return None - opens = self._phase_anchor_date(first_incomplete) - return { - "phase": first_incomplete["id"], - "phase_name": first_incomplete["name"], - "opens": opens.isoformat() if opens else None, - "opens_in_days": (opens - self.as_of).days if opens else None, - } - - def status_report(self) -> dict: - """Structured status — presentation layer (render.py) turns this into a view. - Deterministic; no formatting baked in. Assembled from focused builders: - phase map, current-phase steps, current hold, scheduled window, active phase.""" - (phases, total, done, current_phase_name, - current_phase_obj, current_step_name) = self._phase_map() - current_steps = self._current_steps(current_phase_obj) - - gate = action = None - if self.state.status == "holding_gate" and self.state.current_phase: - gate = self._hold_view(current_phase_name, current_step_name) - elif self.state.status == "awaiting_action" and self.state.current_phase: - # A scout step is the SKILL's work (run via step-action), not a USER action — - # never surface it as `action` (which the digest reads as "Action needed now"). - phase = next((p for p in self.config["phases"] - if p["id"] == self.state.current_phase), None) - cur = next((s for s in (phase or {}).get("steps", []) - if s["id"] == self.state.current_step), None) if self.state.current_step else None - is_scout_focus = bool(cur and cur.get("source") == "scout" and not cur.get("attest")) - cur_blocked = (self.state.get_step(self.state.current_phase, self.state.current_step).status - == "blocked") if self.state.current_step else False - if not is_scout_focus or cur_blocked: - action = self._hold_view(current_phase_name, current_step_name) - scheduled = self._scheduled_view() - - chk = self.gate.checklist() - active_phase = self._active_phase_report() - # Scout steps ready for the SKILL to execute (perform the MCP send/scrape, then - # record-step). They are NOT user holds — the skill drains these itself; only if - # a scout step records 'attention' does it become a blocked user task. - # GATED ON PHASE DUE: a phase that hasn't reached its anchor (e.g. Code Complete - # Day before the CCD) must expose NO pending scout work — otherwise the autonomous - # automation would drain those steps early, running CCD-day comms ahead of the CCD. - # ALSO GATED ON fire_at_local: a timed step (e.g. the 09:00 CCD comms) is excluded - # until its wall-clock time arrives, so the every-hour worker doesn't fire it early - # — its dedicated cron automation runs it at the pinned time. - scout_pending = ([s["id"] for s in (active_phase or {}).get("steps", []) - if s.get("status") == "scout" and s.get("time_ready", True)] - if (active_phase and active_phase.get("due")) else []) - return { - "release_id": self.state.release_id, - "status": self.state.status, - "owner_email": self.state.owner_email, - "owner_name": self.state.owner_name, - "ccd": self.state.ccd, - "ccd_source": self.state.ccd_source, - "ccd_conflict": self.state.ccd_conflict, - "as_of": self.as_of.isoformat(), - "skip_release": self.state.skip_release, - "readiness_signed": self.state.readiness_signed, - "readiness_pending": [i["id"] for i in chk["items"] if not i["satisfied"]], - "blocked": self.state.blocked, - "blocked_items": list(self.state.blocked_items), - "blocked_message": chk.get("blocked_message", ""), - "halted": self.state.halted, - "halt_reason": self.state.halt_reason, - "done": done, "total": total, - "percent": round(100 * done / total) if total else 0, - "phases": phases, - "current_phase": self.state.current_phase, - "current_phase_name": current_phase_name, - "current_step": self.state.current_step, - "current_step_name": current_step_name, - "current_steps": current_steps, - "gate": gate, - "action": action, - "scheduled": scheduled, - "active_phase": active_phase, - "scout_pending": scout_pending, - "pending_human": list(self.state.pending_human), - "gate_decisions": len(self.state.gate_decisions), - "pipeline_runs": dict(getattr(self.state, "pipeline_runs", {}) or {}), - "updated_at": self.state.updated_at, - } - def asdict_gate(gd: GateDecision) -> dict: return {"step": gd.step, "decision": gd.decision, "at": gd.at, diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 7e30208f..ed9bca9c 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -242,6 +242,7 @@ def _pipelines_line(r: dict) -> str: resolved yet. Reads the nested pipeline_runs schema (migrating a legacy flat shape); no live az call in the render path.""" from orchestrator.state import migrate_pipeline_runs + from tools.pipelines import format_versions pr = migrate_pipeline_runs(r.get("pipeline_runs") or {}) parts = [] ch = pr.get("checker") or {} @@ -249,8 +250,7 @@ def _pipelines_line(r: dict) -> str: parts.append(f"checker {ch['run_id']}") o = pr.get("orchestrator") or {} if o.get("run_id"): - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) + vstr = format_versions(o.get("versions")) parts.append(f"orchestrator {o['run_id']}" + (f" ({vstr})" if vstr else "")) rcs = pr.get("rcs") or [] rc = rcs[-1] if rcs else {} diff --git a/release-agent/orchestrator/schedule.py b/release-agent/orchestrator/schedule.py index 384cdc38..326e4d57 100644 --- a/release-agent/orchestrator/schedule.py +++ b/release-agent/orchestrator/schedule.py @@ -33,7 +33,7 @@ def get_tz(name: Optional[str] = None): try: from zoneinfo import ZoneInfo return ZoneInfo(name or DEFAULT_TZ) - except Exception: + except (ImportError, KeyError, ValueError): # no tzdata / unknown zone name return None diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py new file mode 100644 index 00000000..c9efc2e9 --- /dev/null +++ b/release-agent/orchestrator/status_views.py @@ -0,0 +1,264 @@ +"""Status view-model builder — the presentation half of the Orchestrator. + +Extracted from engine.py (which owns the state machine) so the report-model builder is a +separate responsibility. This is a MIXIN on Orchestrator: the methods read engine +internals via self (self.state, self.config, self._phase_due, ...) and return the plain +dict that render.py turns into a view. Behaviour is identical to the in-engine version. +""" +from __future__ import annotations + +from typing import Optional + + +class StatusViewMixin: + def _active_phase_report(self) -> Optional[dict]: + """The first incomplete included phase, with its outstanding steps and + whether its time-window is open (due). This is what the daily phase + notification reports on — independent of state.current_phase (which is + only set once the release has been advanced).""" + for phase in self.config["phases"]: + if not self._phase_included(phase): + continue + steps = phase["steps"] + done = sum(1 for s in steps if self.state.is_done(phase["id"], s["id"])) + if done == len(steps): + continue # phase complete — look at the next one + outstanding = [ + {"id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), "owner": s.get("owner", "agent")} + for s in steps if not self.state.is_done(phase["id"], s["id"]) + ] + completed = [s["name"] for s in steps + if self.state.is_done(phase["id"], s["id"])] + cur = self.state.current_step + steps_view = [] + for s in steps: + sid = s["id"] + stp = self.state.get_step(phase["id"], sid) + s_done = self.state.is_done(phase["id"], sid) + s_blocked = stp.status == "blocked" + is_gate = bool(s.get("gate")) + is_rem = self._is_reminder(s) + is_scout = s.get("source") == "scout" + is_attest = bool(s.get("attest")) + if s_done: + status = "done" + elif s_blocked: + status = "blocked" + elif is_gate: + status = "approval" + elif is_attest: + status = "confirm" + elif is_rem: + status = "action" # a human to-do — the user must act + elif is_scout: + status = "scout" # Scout runs it automatically (scrape/send via MCP) + else: + status = "auto" + # needs_owner = a genuine USER task. A pending scout step is Scout's + # automatic work (not the user's) until it BLOCKS (s_blocked), so it is + # NOT flagged — only gates, reminders, attests, and blocks are. + needs = bool((is_gate or is_rem or is_attest or s_blocked) and not s_done) + steps_view.append({ + "id": sid, "name": s["name"], "status": status, + "needs_owner": needs, + "time_ready": self._step_time_ready(phase, s), # False = waits for its fire_at_local + "note": stp.note, # agent result / block reason / detail + "links": list(getattr(stp, "links", None) or []), # durable refs to items evaluated + "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_attest or s_blocked)), + }) + opens = self._phase_anchor_date(phase) + return { + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase"), + "show_pipeline_runs": bool(phase.get("show_pipeline_runs")), + "done": done, "total": len(steps), + "due": self._phase_due(phase), "started": done > 0, + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + "outstanding": outstanding, + "completed": completed, + "steps": steps_view, + } + return None + + def _phase_map(self): + """Build the phase overview + running totals. Returns + (phases, total, done, current_phase_name, current_phase_obj, current_step_name).""" + phases = [] + total = done = 0 + current_phase_name = current_step_name = None + current_phase_obj = None + for idx, phase in enumerate(self.config["phases"]): + if not self._phase_included(phase): + continue + p_total = len(phase["steps"]) + p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) + total += p_total + done += p_done + is_current = self.state.current_phase == phase["id"] + due = self._phase_due(phase) + opens = self._phase_anchor_date(phase) + if p_total and p_done == p_total: + state = "done" + elif not due and p_done == 0: + state = "scheduled" + elif is_current or p_done > 0: + state = "current" + else: + state = "pending" + if is_current: + current_phase_name = phase["name"] + current_phase_obj = phase + phases.append({ + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase", idx), + "done": p_done, "total": p_total, "state": state, + "current": is_current, + "anchor": phase.get("anchor"), + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + }) + for s in phase["steps"]: + if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: + current_step_name = s["name"] + return phases, total, done, current_phase_name, current_phase_obj, current_step_name + + def _current_steps(self, current_phase_obj) -> list: + """The current phase's steps, each tagged with a display state.""" + if not current_phase_obj: + return [] + phase_due = self._phase_due(current_phase_obj) + out = [] + for s in current_phase_obj["steps"]: + rec = self.state.steps.get(self.state.key(current_phase_obj["id"], s["id"]), {}) or {} + is_scout = s.get("source") == "scout" and not s.get("attest") + if rec.get("status") == "skipped": + s_state = "skipped" + elif self.state.is_done(current_phase_obj["id"], s["id"]): + s_state = "done" + elif rec.get("status") == "blocked": + s_state = "blocked" # a step hit a real problem — needs the owner + elif s["id"] == self.state.current_step and self.state.status == "holding_gate": + s_state = "gate" + elif is_scout: + s_state = "scout" # Scout's automatic work — never a user "do this" + elif s["id"] == self.state.current_step and self.state.status == "awaiting_action": + s_state = "reminder" + elif not phase_due: + s_state = "scheduled" + else: + s_state = "pending" + out.append({ + "id": s["id"], "name": s["name"], + "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), + "owner": s.get("owner", "agent"), + "state": s_state, + "note": rec.get("note"), # agent result / block reason / detail + "links": rec.get("links") or [], # durable refs (wiki page, CG alerts) + }) + return out + + def _hold_view(self, phase_name, step_name) -> dict: + """Detail of the current hold (gate or action-needed) — same shape for both.""" + return { + "phase": self.state.current_phase, + "phase_name": phase_name, + "step": self.state.current_step, + "step_name": step_name, + } + + def _scheduled_view(self) -> Optional[dict]: + """The phase we're waiting on the clock for — derived from the first + incomplete phase's due-ness, so `status` shows it even before `next`.""" + first_incomplete = next( + (p for p in self.config["phases"] + if self._phase_included(p) + and not all(self.state.is_done(p["id"], s["id"]) for s in p["steps"])), + None) + if (first_incomplete is None or self._phase_due(first_incomplete) + or self.state.status in ("complete", "halted", "blocked")): + return None + opens = self._phase_anchor_date(first_incomplete) + return { + "phase": first_incomplete["id"], + "phase_name": first_incomplete["name"], + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + } + + def status_report(self) -> dict: + """Structured status — presentation layer (render.py) turns this into a view. + Deterministic; no formatting baked in. Assembled from focused builders: + phase map, current-phase steps, current hold, scheduled window, active phase.""" + (phases, total, done, current_phase_name, + current_phase_obj, current_step_name) = self._phase_map() + current_steps = self._current_steps(current_phase_obj) + + gate = action = None + if self.state.status == "holding_gate" and self.state.current_phase: + gate = self._hold_view(current_phase_name, current_step_name) + elif self.state.status == "awaiting_action" and self.state.current_phase: + # A scout step is the SKILL's work (run via step-action), not a USER action — + # never surface it as `action` (which the digest reads as "Action needed now"). + phase = next((p for p in self.config["phases"] + if p["id"] == self.state.current_phase), None) + cur = next((s for s in (phase or {}).get("steps", []) + if s["id"] == self.state.current_step), None) if self.state.current_step else None + is_scout_focus = bool(cur and cur.get("source") == "scout" and not cur.get("attest")) + cur_blocked = (self.state.get_step(self.state.current_phase, self.state.current_step).status + == "blocked") if self.state.current_step else False + if not is_scout_focus or cur_blocked: + action = self._hold_view(current_phase_name, current_step_name) + scheduled = self._scheduled_view() + + chk = self.gate.checklist() + active_phase = self._active_phase_report() + # Scout steps ready for the SKILL to execute (perform the MCP send/scrape, then + # record-step). They are NOT user holds — the skill drains these itself; only if + # a scout step records 'attention' does it become a blocked user task. + # GATED ON PHASE DUE: a phase that hasn't reached its anchor (e.g. Code Complete + # Day before the CCD) must expose NO pending scout work — otherwise the autonomous + # automation would drain those steps early, running CCD-day comms ahead of the CCD. + # ALSO GATED ON fire_at_local: a timed step (e.g. the 09:00 CCD comms) is excluded + # until its wall-clock time arrives, so the every-hour worker doesn't fire it early + # — its dedicated cron automation runs it at the pinned time. + scout_pending = ([s["id"] for s in (active_phase or {}).get("steps", []) + if s.get("status") == "scout" and s.get("time_ready", True)] + if (active_phase and active_phase.get("due")) else []) + return { + "release_id": self.state.release_id, + "status": self.state.status, + "owner_email": self.state.owner_email, + "owner_name": self.state.owner_name, + "ccd": self.state.ccd, + "ccd_source": self.state.ccd_source, + "ccd_conflict": self.state.ccd_conflict, + "as_of": self.as_of.isoformat(), + "skip_release": self.state.skip_release, + "readiness_signed": self.state.readiness_signed, + "readiness_pending": [i["id"] for i in chk["items"] if not i["satisfied"]], + "blocked": self.state.blocked, + "blocked_items": list(self.state.blocked_items), + "blocked_message": chk.get("blocked_message", ""), + "halted": self.state.halted, + "halt_reason": self.state.halt_reason, + "done": done, "total": total, + "percent": round(100 * done / total) if total else 0, + "phases": phases, + "current_phase": self.state.current_phase, + "current_phase_name": current_phase_name, + "current_step": self.state.current_step, + "current_step_name": current_step_name, + "current_steps": current_steps, + "gate": gate, + "action": action, + "scheduled": scheduled, + "active_phase": active_phase, + "scout_pending": scout_pending, + "pending_human": list(self.state.pending_human), + "gate_decisions": len(self.state.gate_decisions), + "pipeline_runs": dict(getattr(self.state, "pipeline_runs", {}) or {}), + "updated_at": self.state.updated_at, + } diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 7dcec4a8..58bacdef 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -9,13 +9,19 @@ """ from __future__ import annotations +from datetime import datetime, timezone + # This package is Engineering-only; alias the explicitly-named source constants to the # short local names the step CONFIGs use. (Do NOT use these for msazure/One calls.) +from tools import pipelines as P from tools.pipelines import ( ENGINEERING_ORG as ORG, ENGINEERING_PROJECT as PROJECT, CHECKER_DEF, ORCHESTRATOR_DEF, MRWP_DEF, - ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, + ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, format_versions, ) +from orchestrator.state import migrate_pipeline_runs +from orchestrator.outcomes import Done, Blocked +from steps.lib.mockctx import mock_input, MISSING # Surfaced in every block reason so the engineer knows how to recover / escalate. RECOVERY_TSG = ("https://eng.ms/docs/microsoft-security/identity/" @@ -41,13 +47,11 @@ def links_for(build_id, name="ADO run"): def _now_iso(): - from datetime import datetime, timezone return datetime.now(timezone.utc).isoformat() def _pipeline_runs(state) -> dict: """The nested pipeline_runs container on state (migrating a legacy flat shape).""" - from orchestrator.state import migrate_pipeline_runs return migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) @@ -114,8 +118,6 @@ def rc_report_model(state, timeout=120): iteration (rcs[-1]) and routes through `tools.pipelines.assemble_rc_model` — the SAME assembler the live path uses — so the state-based model can't drift from the live one. """ - from tools import pipelines as P - from orchestrator.state import migrate_pipeline_runs pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) ch = pr.get("checker") or {} o = pr.get("orchestrator") or {} @@ -123,6 +125,9 @@ def rc_report_model(state, timeout=120): rc = rcs[-1] if rcs else {} checker = {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), "when": ch.get("when")} + # healthy=True is true by construction here: rc_report only runs AFTER orchestrator_health + # passed (a failed pre-gate stage blocks that step, so we never reach this with an + # unhealthy orchestrator). The live path (release_report) derives it from stages. orchestrator = {"found": bool(o.get("run_id")), "healthy": True, "run_id": o.get("run_id"), "versions": o.get("versions") or {}, "parked": o.get("parked")} @@ -257,14 +262,25 @@ def _fail_rate(failed, total) -> float: # Test-run category labels (mirrors tools.pipelines.classify_test_run). _CAT_LABEL = {"unit": "Unit", "instrumented": "Instrumented", "ui": "UI automation"} +# Failing-suite display order: UI first (the RC-critical bucket), then instrumented, unit. +_SUITE_ORDER = {"ui": 0, "instrumented": 1, "unit": 2} + + +def sort_failed_suites(suites): + """Failing suites ordered UI-first then instrumented/unit, each by descending failure + count. One helper so every RC renderer (plain email, HTML email, CLI report) lists + them identically.""" + return sorted(suites or [], + key=lambda s: (_SUITE_ORDER.get(s.get("category", "ui"), 9), + -(s.get("failed") or 0))) + def _rc_email_plain(model, ctx) -> str: """Plain-text form of the RC report email (fallback + logging).""" L = [] rid = model.get("release", "?") o = model.get("orchestrator") or {} - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or "n/a" + vstr = format_versions(o.get("versions"), fallback="n/a") L.append(f"Hi {ctx.get('owner', 'there')},") L.append("") L.append(f"The Release Candidate for {rid} has been built and RC testing has " @@ -297,9 +313,7 @@ def _rc_email_plain(model, ctx) -> str: fs = r.get("failed_stages") or [] if fs: L.append(f" Red stages ({len(fs)}): {', '.join(fs)}") - _ord = {"ui": 0, "instrumented": 1, "unit": 2} - for s in sorted((r.get("failed_suites") or []), - key=lambda s: (_ord.get(s.get("category", "ui"), 9), -s["failed"])): + for s in sort_failed_suites(r.get("failed_suites")): sr = _fail_rate(s["failed"], s["total"]) L.append(f" [{_CAT_LABEL.get(s.get('category', 'ui'), 'UI automation')}] " f"{s['name']} — {s['failed']}/{s['total']} failed ({sr}%):") @@ -337,8 +351,7 @@ def _rc_email_html(model, ctx) -> str: from steps.lib import templating as T rid = model.get("release", "?") o = model.get("orchestrator") or {} - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or "n/a" + vstr = format_versions(o.get("versions"), fallback="n/a") ch = model.get("checker") or {} park = ("parked at ‘Remove RC Tags’" if o.get("parked") else "gate cleared") @@ -392,9 +405,7 @@ def mrwp_card(prov): ("unit", "instrumented", "ui")) # Failing suites — UI first, then instrumented/unit; each tagged by category. - _ord = {"ui": 0, "instrumented": 1, "unit": 2} - suites = sorted((r.get("failed_suites") or []), - key=lambda s: (_ord.get(s.get("category", "ui"), 9), -s["failed"])) + suites = sort_failed_suites(r.get("failed_suites")) suite_html = "" for s in suites: sr = _fail_rate(s["failed"], s["total"]) @@ -540,10 +551,6 @@ def verify_mrwp(state, provider): tests : inject the test summary {total,passed,failed[,runs]} Returns a Done/Blocked outcome. """ - from orchestrator.outcomes import Done, Blocked - from steps.lib.mockctx import mock_input, MISSING - from tools import pipelines as P - label = f"MRWP {provider}" # 1) resolve the MRWP build id for this provider mid = mock_input("mrwp_id", MISSING) diff --git a/release-agent/steps/build_verify/orchestrator_health.py b/release-agent/steps/build_verify/orchestrator_health.py index 82a079c2..0059e8b4 100644 --- a/release-agent/steps/build_verify/orchestrator_health.py +++ b/release-agent/steps/build_verify/orchestrator_health.py @@ -57,7 +57,7 @@ def build(state): links = K.links_for(bid, "Release Orchestrator run") versions = {k: P._tag_value(tags, f"Next{k}Version") for k in ("Common", "Msal", "Broker")} - vstr = ", ".join(f"{k} {v}" for k, v in versions.items() if v) or "versions n/a" + vstr = P.format_versions(versions, fallback="versions n/a") stages = mock_input("stages", MISSING) if stages is MISSING: diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 9df8b9a5..adf02aec 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -300,6 +300,16 @@ def stage_completion(stages): TEST_CATEGORIES = ("unit", "instrumented", "ui") _CATEGORY_LABEL = {"unit": "Unit", "instrumented": "Instrumented", "ui": "UI automation"} +_VERSION_KEYS = ("Common", "Msal", "Broker") + + +def format_versions(versions, fallback: str = "") -> str: + """'Common X, Msal Y, Broker Z' from a {Common,Msal,Broker} dict — fixed order, + blanks omitted. Returns `fallback` when nothing is set. One place so every report / + status render formats RC versions identically.""" + v = versions or {} + return ", ".join(f"{k} {v[k]}" for k in _VERSION_KEYS if v.get(k)) or fallback + def classify_test_run(name): """Bucket a test-run/suite name into one of THREE categories: From ea8ce7d62f84ebfab53c1f21e053870f84fc8ba0 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 23:16:52 +0100 Subject: [PATCH 71/90] =?UTF-8?q?release-agent:=20status=20table=20?= =?UTF-8?q?=E2=80=94=20show=20engine-run=20agent=20steps=20as=20automatic,?= =?UTF-8?q?=20not=20'Pending'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UX fix: the four Phase-2 verification steps are agent steps Scout runs itself in-process, but they rendered as 'Pending' (⬜) next to the scout rc_report step's 'Scout runs this — automatic' (🤖) — a confusing mixed message implying the human was waiting on the agent steps. New 'auto' display state for engine-run agent steps renders IDENTICALLY to the skill-run 'scout' state (🤖 'Scout runs this — automatic'). 'Pending' is now reserved for a human step queued behind a dependency, so it always means 'you'll act here'. Everything needing the user stays distinct (⏸ gate, 📌 reminder, ⛔ blocked). Render-only; no state-machine change. 189 tests pass (added a display-contract regression test). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/render.py | 10 +++++--- release-agent/orchestrator/status_views.py | 4 +++- release-agent/tests/test_engine.py | 27 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index ed9bca9c..65687902 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -32,7 +32,7 @@ def step_detail(s: dict, limit: int = 160) -> str: links = s.get("links") or [] if not note and not links: state = s.get("state") - return "—" if state in ("pending", "scheduled", "reminder", "gate") else "" + return "—" if state in ("pending", "scheduled", "reminder", "gate", "auto") else "" text = str(note or "").strip() lead = next((ln.strip() for ln in text.splitlines() if ln.strip()), "") # Prefer STRUCTURED links (first-class, stored on state) over URL-in-prose. @@ -229,10 +229,14 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: } _PHASE_ICON = {"done": "✅", "current": "⏸", "pending": "⬜", "scheduled": "🗓"} _STEP_ICON = {"done": "✅", "gate": "⏸", "reminder": "📌", "scheduled": "🗓", - "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "blocked": "⛔"} + "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "auto": "🤖", "blocked": "⛔"} _STEP_STATE_WORD = {"done": "Done", "gate": "Awaiting your approval", "reminder": "Do this — then mark done", "scheduled": "Not open yet", "pending": "Pending", "skipped": "Skipped", + # 'auto' (engine-run agent step) and 'scout' (skill-run MCP step) are BOTH + # Scout's automatic work — shown identically so the human sees one message + # (no action needed), never a confusing "Pending" next to "automatic". + "auto": "Scout runs this — automatic", "scout": "Scout runs this — automatic", "blocked": "Blocked — needs you"} @@ -410,7 +414,7 @@ def _phase_num(r: dict, phase_id: str): def _next_pending_step(r: dict): for s in r.get("current_steps", []): - if s["state"] in ("pending", "gate"): + if s["state"] in ("pending", "gate", "auto"): return s["name"] return None diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py index c9efc2e9..2cbddb65 100644 --- a/release-agent/orchestrator/status_views.py +++ b/release-agent/orchestrator/status_views.py @@ -147,8 +147,10 @@ def _current_steps(self, current_phase_obj) -> list: s_state = "reminder" elif not phase_due: s_state = "scheduled" + elif s.get("owner") == "human" or s.get("attest") or self._is_reminder(s): + s_state = "pending" # a human step queued behind a dependency else: - s_state = "pending" + s_state = "auto" # an agent step Scout runs itself — no user action out.append({ "id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 41add017..38c42d33 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2189,6 +2189,33 @@ def test_active_phase_report_steps_carry_links(): assert ap and all("links" in s for s in ap["steps"]) +def test_agent_steps_render_as_automatic_not_pending(): + """UX contract: engine-run agent steps ('auto') read IDENTICALLY to skill-run scout + steps — 'Scout runs this — automatic', 🤖 — so a human never sees a confusing + 'Pending' next to 'automatic' for work that needs no action. 'Pending' stays distinct + (a human step still to come).""" + from orchestrator import render + assert render._STEP_STATE_WORD["auto"] == render._STEP_STATE_WORD["scout"] == \ + "Scout runs this — automatic" + assert render._STEP_ICON["auto"] == render._STEP_ICON["scout"] == "🤖" + assert render._STEP_STATE_WORD["pending"] != render._STEP_STATE_WORD["auto"] + + # And the classifier tags the (pending, engine-run) build_verify agent steps as 'auto', + # the scout email as 'scout', never 'pending'. Force Phases 0-1 done so build_verify is + # the current phase with its steps still pending. + from orchestrator.state import StepState + st, orch = _mock_orch({}, as_of="2026-07-09") # build_verify anchor CCD+1 → due + for pid in ("preflight", "ccd"): + ph = next(p for p in orch.config["phases"] if p["id"] == pid) + for s in ph["steps"]: + orch.state.set_step(pid, s["id"], StepState(status="done", by="test")) + orch.state.current_phase = "build_verify" # the engine sets this during `next` + steps = {s["id"]: s for s in orch.status_report()["current_steps"]} + for sid in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): + assert steps[sid]["state"] == "auto", (sid, steps[sid]["state"]) + assert steps["rc_report"]["state"] == "scout" + + def test_reconcile_retries_pure(): """reconcile_retries collapses per-attempt results by title: passed if any attempt passed; recovered if it also failed; failed only if it never passed; NA ignored.""" From 40a3641daab1fe605a491d2c901f44c58e31dfb5 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 23:34:48 +0100 Subject: [PATCH 72/90] =?UTF-8?q?release-agent:=20phase=20map=20=E2=80=94?= =?UTF-8?q?=20only=20the=20frontier=20shows=20as=20'current'=20(no=20false?= =?UTF-8?q?=202nd=20active=20phase)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: _phase_map marked ANY phase with done>0 as 'current' (⏸). After an upstream reopen left stale progress in a later phase, TWO phases rendered as 'in progress' — e.g. Phase 2 (frontier, 4/5) AND Phase 3 (2/5 leftover). Confusing: looked like Phase 3 ran while Phase 2 was incomplete. Fix: derive the frontier (first-incomplete included phase) once and mark ONLY it 'current'; a later phase with stale partial progress renders 'pending' (its count still shows). Ordering preserved so a not-yet-due frontier stays 'scheduled'. current_phase_obj/current_step_name are now frontier-derived too (equal to the engine cursor during real holds — no behaviour change). Display-only; the engine already processed the frontier first and never re-ran downstream. 190 tests pass (added a regression reproducing the two-active-phase state). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/status_views.py | 18 +++++++++++++----- release-agent/tests/test_engine.py | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py index 2cbddb65..1aed4fdc 100644 --- a/release-agent/orchestrator/status_views.py +++ b/release-agent/orchestrator/status_views.py @@ -89,6 +89,13 @@ def _phase_map(self): total = done = 0 current_phase_name = current_step_name = None current_phase_obj = None + # The authoritative CURRENT phase is the FRONTIER — the first included phase with + # incomplete steps — NOT merely "any phase with progress". Deriving it here (rather + # than trusting p_done > 0) means exactly ONE phase shows as current, even when an + # upstream reopen left stale progress in a later phase (which would otherwise render + # as a confusing second "in progress" phase). + frontier = self._current_phase() + frontier_id = frontier["id"] if frontier else None for idx, phase in enumerate(self.config["phases"]): if not self._phase_included(phase): continue @@ -96,17 +103,18 @@ def _phase_map(self): p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) total += p_total done += p_done - is_current = self.state.current_phase == phase["id"] + is_current = phase["id"] == frontier_id due = self._phase_due(phase) opens = self._phase_anchor_date(phase) if p_total and p_done == p_total: state = "done" elif not due and p_done == 0: - state = "scheduled" - elif is_current or p_done > 0: + state = "scheduled" # not open yet — even if it's the frontier + elif is_current: state = "current" else: - state = "pending" + state = "pending" # not the frontier — a later phase, even if it has + # stale partial progress, is not "in progress" now if is_current: current_phase_name = phase["name"] current_phase_obj = phase @@ -120,7 +128,7 @@ def _phase_map(self): "opens_in_days": (opens - self.as_of).days if opens else None, }) for s in phase["steps"]: - if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: + if s["id"] == self.state.current_step and phase["id"] == frontier_id: current_step_name = s["name"] return phases, total, done, current_phase_name, current_phase_obj, current_step_name diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 38c42d33..5b741308 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -974,6 +974,28 @@ def test_phase0_opens_on_ccd_minus_7(): assert st.is_done("preflight", "notice") +def test_only_frontier_phase_shows_current_despite_stale_downstream_progress(): + """Issue-B regression: exactly ONE phase (the frontier = first incomplete) renders as + 'current'. A LATER phase left with stale partial progress (e.g. after reopening an + upstream phase) must NOT also show as 'current' — it's 'pending'.""" + from orchestrator.state import StepState + st, orch = _mock_orch({}, as_of="2026-07-09") + # Phases 0-1 done; build_verify (Phase 2) incomplete = the frontier; bug_bash (Phase 3) + # carries stale progress (2 steps done) as if an upstream reopen rolled Phase 2 back. + for pid in ("preflight", "ccd"): + for s in next(p for p in orch.config["phases"] if p["id"] == pid)["steps"]: + orch.state.set_step(pid, s["id"], StepState(status="done", by="test")) + for sid in ("clone_plans", "coordinate"): + orch.state.set_step("bug_bash", sid, StepState(status="done", by="test")) + phases = {p["id"]: p for p in orch.status_report()["phases"]} + assert phases["build_verify"]["state"] == "current" and phases["build_verify"]["current"] + assert phases["bug_bash"]["state"] == "pending" # stale progress ≠ a 2nd current + assert phases["bug_bash"]["done"] == 2 # the leftover count still shows + assert not phases["bug_bash"]["current"] + # exactly one phase is current + assert sum(1 for p in phases.values() if p["state"] == "current") == 1 + + def test_phase_map_marks_scheduled(): st, orch = _ccd_orch("2026-06-28") orch.run_until_gate() From 44ed6ae7c731ab0a3d577b54b98a2dc63227ce10 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:32:27 +0100 Subject: [PATCH 73/90] Phase 2: status-aware RC verify + in-flight state (no false-block on in-progress runs) An MRWP RC run that is still notStarted/inProgress no longer blocks as an aborted pipeline. verify_mrwp now checks the run's overall status first (new pipelines.get_build_status) and returns a new InProgress outcome when it isn't completed - a pending stage on a live run means "not run YET", not aborted. Engine records this as a new non-blocking StepState status "in_flight": not added to pending_human, release stays running (no user action), drain returns a "waiting" hold, and in_flight_since is stamped for the upcoming 6h poller nudge. The step stays not-done so it re-runs on every next/poll until the run completes, then the normal gate applies. Display: in_flight renders as "RC running - Scout is polling" in CLI status + email digest. Adds 2 tests; suite 192 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/engine.py | 19 +++++++++++- release-agent/orchestrator/outcomes.py | 15 ++++++++++ release-agent/orchestrator/render.py | 9 ++++-- release-agent/orchestrator/state.py | 2 +- release-agent/orchestrator/status_views.py | 5 ++++ release-agent/phases/stub_runner.py | 2 ++ release-agent/steps/build_verify/_common.py | 17 ++++++++++- release-agent/steps/lib/agent.py | 12 +++++--- release-agent/tests/test_engine.py | 33 +++++++++++++++++++++ release-agent/tools/pipelines.py | 19 ++++++++++++ 10 files changed, 124 insertions(+), 9 deletions(-) diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index ae4ac52d..fa86c654 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -423,6 +423,23 @@ def _run_auto_step(self, phase: dict, step: dict, block_holds: bool) -> NextActi with mockctx.active(self.mocks.get(f"{pid}.{step['id']}", {})): result = runner(pid, step, self.state) key = f"{pid}.{step['id']}" + # IN-FLIGHT: the step's underlying pipeline run is still executing — NOT a failure. + # Hold the phase as 'waiting on the pipeline' (no user action) and let the poller / + # tick re-run the step until the run completes. Stamp when we first saw it in-flight + # so a poller can send the 6h courtesy nudge. + if getattr(result, "in_flight", False): + prev = self.state.get_step(pid, step["id"]) + data = dict(getattr(prev, "data", {}) or {}) + data.setdefault("in_flight_since", _now()) + data["poll_in_min"] = getattr(result, "poll_in_min", 30) + self.state.set_step(pid, step["id"], + StepState(status="in_flight", note=result.action, by="agent", + links=list(getattr(result, "links", None) or []), + data=data)) + self.state.pending_human = [p for p in self.state.pending_human if p != key] + self.state.status = "running" + return NextAction(kind="waiting", phase=pid, step=step["id"], name=step["name"], + message=f"WAITING — {step['name']}: {result.action}") if not result.ok: self.state.set_step(pid, step["id"], StepState(status="blocked", note=result.action, by=result.by, @@ -455,7 +472,7 @@ def run_until_gate(self, max_steps: int = 500) -> list: for _ in range(max_steps): act = self.step_once(attempted) actions.append(act) - if act.kind in ("gate", "reminder", "scheduled", "complete", "readiness", "blocked", "halted"): + if act.kind in ("gate", "reminder", "scheduled", "complete", "readiness", "blocked", "halted", "waiting"): break return actions diff --git a/release-agent/orchestrator/outcomes.py b/release-agent/orchestrator/outcomes.py index 6106b157..84becb00 100644 --- a/release-agent/orchestrator/outcomes.py +++ b/release-agent/orchestrator/outcomes.py @@ -41,6 +41,21 @@ class Blocked: kind: str = "blocked" +@dataclass +class InProgress: + """An agent step whose underlying work is STILL RUNNING (not a failure, not done). + + Used by the Phase-2 MRWP verification when the RC pipeline run's overall status is + notStarted/inProgress: the step must NOT block as 'aborted' (a never-ran stage during + an in-flight run is just not-run-YET). The engine holds the phase as 'waiting on the + pipeline' — no user action — and a poller re-runs the step every `poll_in_min` minutes + until the run completes, at which point the normal Done/Blocked rules apply.""" + note: str = "" + links: list = field(default_factory=list) + poll_in_min: int = 30 + kind: str = "in_progress" + + @dataclass class NeedsHuman: prompt: str diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 65687902..88558c74 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -229,7 +229,8 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: } _PHASE_ICON = {"done": "✅", "current": "⏸", "pending": "⬜", "scheduled": "🗓"} _STEP_ICON = {"done": "✅", "gate": "⏸", "reminder": "📌", "scheduled": "🗓", - "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "auto": "🤖", "blocked": "⛔"} + "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "auto": "🤖", "blocked": "⛔", + "in_flight": "⏳"} _STEP_STATE_WORD = {"done": "Done", "gate": "Awaiting your approval", "reminder": "Do this — then mark done", "scheduled": "Not open yet", "pending": "Pending", "skipped": "Skipped", @@ -237,7 +238,10 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: # Scout's automatic work — shown identically so the human sees one message # (no action needed), never a confusing "Pending" next to "automatic". "auto": "Scout runs this — automatic", - "scout": "Scout runs this — automatic", "blocked": "Blocked — needs you"} + "scout": "Scout runs this — automatic", "blocked": "Blocked — needs you", + # in_flight = the pipeline run is still executing. No user action — Scout + # is polling every 30 min and re-evaluates when the run completes. + "in_flight": "RC running — Scout is polling"} def _pipelines_line(r: dict) -> str: @@ -559,6 +563,7 @@ def _esc(s: str) -> str: "action": ("Your action", "#b54708", "#fffaeb"), "scout": ("Scout runs this", "#475467", "#f2f4f7"), "auto": ("Automatic — pending", "#475467", "#f2f4f7"), + "in_flight": ("⏳ RC running — polling", "#475467", "#f2f4f7"), } diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py index 5a9389e1..513da2d5 100644 --- a/release-agent/orchestrator/state.py +++ b/release-agent/orchestrator/state.py @@ -74,7 +74,7 @@ def migrate_pipeline_runs(pr) -> dict: @dataclass class StepState: """Persisted state for a single step.""" - status: str = "pending" # pending | done | skipped | blocked + status: str = "pending" # pending | done | skipped | blocked | in_flight completed_at: Optional[str] = None note: Optional[str] = None by: Optional[str] = None # 'agent' (stub) or 'human' diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py index 1aed4fdc..ce20b6ab 100644 --- a/release-agent/orchestrator/status_views.py +++ b/release-agent/orchestrator/status_views.py @@ -37,6 +37,7 @@ def _active_phase_report(self) -> Optional[dict]: stp = self.state.get_step(phase["id"], sid) s_done = self.state.is_done(phase["id"], sid) s_blocked = stp.status == "blocked" + s_inflight = stp.status == "in_flight" is_gate = bool(s.get("gate")) is_rem = self._is_reminder(s) is_scout = s.get("source") == "scout" @@ -45,6 +46,8 @@ def _active_phase_report(self) -> Optional[dict]: status = "done" elif s_blocked: status = "blocked" + elif s_inflight: + status = "in_flight" # pipeline run still executing — Scout polling elif is_gate: status = "approval" elif is_attest: @@ -147,6 +150,8 @@ def _current_steps(self, current_phase_obj) -> list: s_state = "done" elif rec.get("status") == "blocked": s_state = "blocked" # a step hit a real problem — needs the owner + elif rec.get("status") == "in_flight": + s_state = "in_flight" # pipeline run still executing — Scout is polling elif s["id"] == self.state.current_step and self.state.status == "holding_gate": s_state = "gate" elif is_scout: diff --git a/release-agent/phases/stub_runner.py b/release-agent/phases/stub_runner.py index 0081b528..8dc43d08 100644 --- a/release-agent/phases/stub_runner.py +++ b/release-agent/phases/stub_runner.py @@ -19,6 +19,8 @@ class StepResult: action: str # human-readable description of what happened / should happen by: str # 'agent' (stub did it) or 'human' (needs a person) links: list = None # optional [{name, url}] durable refs (wiki page, CG alerts) + in_flight: bool = False # True → the work is still RUNNING (poll again); not done, not blocked + poll_in_min: int = 30 # re-check cadence when in_flight def run_stub(phase_id: str, step: dict, state=None) -> StepResult: diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 58bacdef..ba790f51 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -20,7 +20,7 @@ ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, format_versions, ) from orchestrator.state import migrate_pipeline_runs -from orchestrator.outcomes import Done, Blocked +from orchestrator.outcomes import Done, Blocked, InProgress from steps.lib.mockctx import mock_input, MISSING # Surfaced in every block reason so the engineer knows how to recover / escalate. @@ -575,6 +575,21 @@ def verify_mrwp(state, provider): links = links_for(mid, f"{label} run") + # 1.5) overall run status — an in-flight run is NOT a failure. If the MRWP run is + # still notStarted/inProgress, its un-run stages just haven't run YET; hold the step + # as in-flight and let the 30-min poller re-evaluate when it completes, instead of + # false-blocking it as an aborted release. `build_status` mock drives this in sim/tests; + # when stages are injected (no live call) we assume the run is complete. + bstatus = mock_input("build_status", MISSING) + if bstatus is MISSING and mock_input("stages", MISSING) is MISSING: + ok_s, bstatus, _bres, _bdetail = P.get_build_status(ORG, PROJECT, mid) + if not ok_s: + bstatus = None # status unknown → fall through to the stage rule (best-effort) + if bstatus not in (MISSING, None) and bstatus != "completed": + return InProgress( + f"{label} run {mid} is still running (status: {bstatus}) — Scout is polling " + f"every 30 min and will re-evaluate the RC when it completes.", links=links) + # 2) stage-completion rule stages = mock_input("stages", MISSING) if stages is MISSING: diff --git a/release-agent/steps/lib/agent.py b/release-agent/steps/lib/agent.py index 07d1b4c7..9223e140 100644 --- a/release-agent/steps/lib/agent.py +++ b/release-agent/steps/lib/agent.py @@ -15,21 +15,25 @@ """ from __future__ import annotations -from orchestrator.outcomes import Done, Blocked +from orchestrator.outcomes import Done, Blocked, InProgress from phases.stub_runner import StepResult def to_step_result(outcome) -> StepResult: - """Map a uniform Outcome to the engine's StepResult (agent steps only ever - return Done/Blocked — never NeedsSkill/NeedsHuman).""" + """Map a uniform Outcome to the engine's StepResult. Agent steps return Done/Blocked, + or InProgress when their underlying pipeline run is still executing (poll again).""" if isinstance(outcome, Done): return StepResult(ok=True, action=outcome.note, by=outcome.by, links=list(outcome.links or [])) if isinstance(outcome, Blocked): return StepResult(ok=False, action=outcome.reason, by="agent", links=list(outcome.links or [])) + if isinstance(outcome, InProgress): + return StepResult(ok=False, action=outcome.note, by="agent", + links=list(outcome.links or []), + in_flight=True, poll_in_min=outcome.poll_in_min) raise TypeError( - f"agent step returned {type(outcome).__name__}; expected Done or Blocked") + f"agent step returned {type(outcome).__name__}; expected Done/Blocked/InProgress") def legacy_run(build): diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 5b741308..0a740af9 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2005,6 +2005,39 @@ def test_build_verify_steps_pass_with_healthy_mocks(): assert "Tests:" in outs["mrwp_ecs"]["note"] and "1 red" in outs["mrwp_ecs"]["note"] +def test_build_verify_mrwp_in_flight_when_run_still_executing(): + """An MRWP run whose OVERALL status is still inProgress is NOT a failure — verify + returns in_progress (Scout polls + re-evaluates on completion) instead of blocking it + as an aborted pipeline. A pending stage on an in-flight run is 'not run YET'.""" + st, orch = _bv_state({"build_verify.mrwp_ecs": { + "mrwp_id": "999", "build_status": "inProgress"}}) + out = _bv_build(orch, st, "mrwp_ecs") + assert out["kind"] == "in_progress" + assert "still running" in out["note"] and "poll" in out["note"].lower() + assert out["poll_in_min"] == 30 + # a completed run still runs the normal rule (control): injected stages, no in-flight + st2, orch2 = _bv_state({}) + assert _bv_build(orch2, st2, "mrwp_ecs")["kind"] == "done" + + +def test_engine_in_flight_step_holds_without_flagging_owner(): + """The engine records an in-flight agent step as status 'in_flight' (not blocked): + it is NOT added to pending_human, the release stays 'running' (no user action), the + drain returns a 'waiting' action, and first-seen time is stamped for the 6h nudge.""" + st, orch = _bv_state({"build_verify.mrwp_ecs": { + "mrwp_id": "999", "build_status": "inProgress"}}) + phase = next(p for p in orch.config["phases"] if p["id"] == "build_verify") + step = next(s for s in phase["steps"] if s["id"] == "mrwp_ecs") + act = orch._run_auto_step(phase, step, block_holds=True) + assert act.kind == "waiting" + rec = st.get_step("build_verify", "mrwp_ecs") + assert rec.status == "in_flight" + assert "build_verify.mrwp_ecs" not in st.pending_human + assert st.status == "running" + assert rec.data.get("in_flight_since") and rec.data.get("poll_in_min") == 30 + assert not st.is_done("build_verify", "mrwp_ecs") # not done → re-runs on next poll + + def test_build_verify_mrwp_blocks_on_never_ran_stage(): """An MRWP run with a skipped/pending stage blocks (aborted pipeline), with the recovery TSG + escalation in the reason.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index adf02aec..7853c9be 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -260,6 +260,25 @@ def named_record(records, name, types=("Job", "Phase", "Stage")): return None +def get_build_status(org, project, build_id, timeout=60): + """Return (ok, status, result, detail) for a build's OVERALL run. + + status ∈ {notStarted, inProgress, completed, cancelling, postponed, none} + result ∈ {succeeded, partiallySucceeded, failed, canceled, none} (only meaningful + once status == 'completed'). + + This is the Phase-2 completion signal: a run is DONE only when status == 'completed'. + While it's notStarted/inProgress the verify step must treat un-run stages as + 'not run YET' (in-flight), NOT as an aborted release.""" + ok, data, detail = _az_json( + ["pipelines", "build", "show", "--org", org, "--project", project, + "--id", str(build_id), "--query", "{status:status,result:result}"], timeout) + if not ok: + return (False, None, None, detail) + d = data or {} + return (True, d.get("status"), d.get("result"), "") + + def get_stages(org, project, build_id, timeout=60): """Return (ok, stages, detail). `stages` is an ORDER-sorted list of {name, state, result} from the build's timeline (Stage records only).""" From d0624515b4d55a91f66fdc59e4e0c16e64822ba7 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:36:15 +0100 Subject: [PATCH 74/90] Phase 2: `rc-retriggered` signal command (reopen RC steps for the newest RC) When the human explicitly signals a NEW RC was triggered - a flaky-suite re-run, or the orchestrator re-running RC testing after a broker cherry-pick - `rc-retriggered --release [--reason]` reopens the two MRWP verifies + rc_report (checker_fired / orchestrator_health are left intact; a re-triggered RC re-runs MRWP against the same orchestrator run). mrwp_run_ids already picks the highest (newest) run id, so the engine re-resolves the new RC and re-applies the gate on the next tick/poll; the status-aware verify holds while it is still in-flight, so an early poll cannot false-fail an in-progress RC. Clears the reopened steps from pending_human, flips status back to running, and journals an `rc_retriggered` event. Adds a test; suite 193 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../orchestrator/commands/release.py | 50 +++++++++++++++++++ release-agent/tests/test_engine.py | 30 +++++++++++ 2 files changed, 80 insertions(+) diff --git a/release-agent/orchestrator/commands/release.py b/release-agent/orchestrator/commands/release.py index db422ee0..478fea2c 100644 --- a/release-agent/orchestrator/commands/release.py +++ b/release-agent/orchestrator/commands/release.py @@ -195,6 +195,47 @@ def cmd_reopen(args): return 0 +# The Phase-2 RC-testing steps a re-triggered RC invalidates: the two MRWP verifications +# and the terminal RC report/gate. checker_fired / orchestrator_health are NOT reopened — +# a re-triggered RC re-runs MRWP against the same orchestrator run. +_RC_RETRIGGER_STEPS = ("mrwp_ecs", "mrwp_local", "rc_report") + + +def cmd_rc_retriggered(args): + """The human explicitly signals that a NEW RC has been triggered (a flaky-run + re-trigger, or the orchestrator re-running RC testing after a broker cherry-pick). + + Reopens the Phase-2 RC-testing steps so the engine re-resolves the NEWEST MRWP run + (mrwp_run_ids already picks the highest id) and re-applies the gate. Scout's poller / + next then holds while the new run is in-flight and re-evaluates on completion — so an + early poll can't mark an in-progress RC as a false failure.""" + st, orch = C.load_orch(args.runs_root, args.release, args.config) + reason = (args.reason or "RC re-triggered").strip() + reopened = [] + for sid in _RC_RETRIGGER_STEPS: + act = orch.reopen_step("build_verify", sid, reason) + if act.kind != "idle": + reopened.append(sid) + # a reopened step is no longer an owner action / block + key = f"build_verify.{sid}" + st.pending_human = [p for p in st.pending_human if p != key] + if not reopened: + print("No Phase-2 RC steps found to reopen (is this release in Build & RC " + "Verification?).") + return 1 + if st.status in ("awaiting_action", "holding_gate", "complete", "halted"): + st.status = "running" + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "rc_retriggered", driver=reason, steps=",".join(reopened)) + msg = (f"RC re-trigger acknowledged — reopened {', '.join(reopened)}. Scout will " + f"re-resolve the newest RC and re-apply the gate; it holds (no action needed) " + f"while the run is still in-flight and polls every 30 min. Reason: {reason}") + C.emit(args.runs_root, args.release, msg, kind="override") + print(msg) + return 0 + + def cmd_halt(args): st, orch = C.load_orch(args.runs_root, args.release, args.config) act = orch.halt(args.reason or "") @@ -293,6 +334,15 @@ def register(sub): ro.add_argument("--reason", default="", help="Why (optional)") ro.set_defaults(func=cmd_reopen) + rt = sub.add_parser("rc-retriggered", + help="Signal a NEW RC was triggered — reopens the Phase-2 RC steps " + "so Scout re-evaluates the newest RC (holds while in-flight)") + rt.add_argument("--release", required=True) + rt.add_argument("--reason", default="", + help="Why it was re-triggered (e.g. 'flaky broker suite re-run' or " + "'broker cherry-pick #123') — recorded for audit") + rt.set_defaults(func=cmd_rc_retriggered) + ht = sub.add_parser("halt", help="Emergency hold — nothing advances until resume (reason REQUIRED)") ht.add_argument("--release", required=True) ht.add_argument("--reason", required=True, help="Why (audit — required)") diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 0a740af9..36f97203 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -749,6 +749,36 @@ def test_reopen_step(): assert st.current_step == "bash_done" # gate re-holds +def test_rc_retriggered_reopens_phase2_rc_steps(): + """`rc-retriggered` reopens the two MRWP verifies + rc_report so Scout re-evaluates the + NEWEST RC, clears them from pending_human, and flips status back to running — while + leaving checker_fired / orchestrator_health (which a re-triggered RC doesn't invalidate) + untouched.""" + import tempfile, argparse + from orchestrator.commands import release as R + from orchestrator.state import StepState + from orchestrator import cli_common as _C + st = ReleaseState(release_id="2026-08", ccd="2026-08-26", ccd_source="confirmed") + for sid in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): + st.set_step("build_verify", sid, StepState(status="done")) + st.set_step("build_verify", "rc_report", StepState(status="blocked", note="UI 88%")) + st.pending_human = ["build_verify.rc_report"] + st.status = "awaiting_action" + with tempfile.TemporaryDirectory() as d: + _C.save_state(st, d, "2026-08") + ns = argparse.Namespace(runs_root=d, release="2026-08", config=CONFIG, + reason="flaky broker suite re-run") + assert R.cmd_rc_retriggered(ns) == 0 + again = _C.load_state(d, "2026-08") + assert not again.is_done("build_verify", "mrwp_ecs") + assert not again.is_done("build_verify", "mrwp_local") + assert again.get_step("build_verify", "rc_report").status == "pending" + assert again.is_done("build_verify", "checker_fired") # untouched + assert again.is_done("build_verify", "orchestrator_health") # untouched + assert "build_verify.rc_report" not in again.pending_human + assert again.status == "running" + + def test_halt_blocks_then_resume(): st, orch = _orch() orch.halt("prod incident") From 012bad9dd273ec8ae65f726738a604e12dbb891b Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:41:29 +0100 Subject: [PATCH 75/90] Phase 2: 30-min RC poller + 6h courtesy nudge (poll-rc + on-demand automation) Adds the poller that watches an in-flight re-triggered RC and drives it to a verdict: * poll-rc command: advances the drain (in-flight verify re-checks the run's live status), then emits a deterministic decision - waiting / nudge / resolved / blocked / idle. A run in-flight past NUDGE_AFTER_HOURS (6h) yields ONE courtesy heads-up to the owner (email + Teams text), stamped nudged_at so it never repeats. Not a failure - Scout keeps polling. * build-verify-rc-poller automation (config/automations.yaml): a 30-min interval poller driving build_verify.rc_report, marked on_demand so it is provisioned ONLY when an RC is re-triggered (not at release start) and torn down when poll-rc reports resolved. plan() now surfaces on_demand. * rc_report.automation_prompt: the bespoke poll instruction (act on the decision; deregister on resolved), owned by the step like localization's. Adds 3 tests (waiting->nudge->dedupe, resolved/blocked/idle, plan shape); suite 196 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/automations.yaml | 14 ++ release-agent/orchestrator/automations.py | 4 + .../orchestrator/commands/__init__.py | 3 +- .../orchestrator/commands/rc_poll.py | 129 ++++++++++++++++++ release-agent/steps/build_verify/rc_report.py | 26 ++++ release-agent/tests/test_engine.py | 80 +++++++++++ 6 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 release-agent/orchestrator/commands/rc_poll.py diff --git a/release-agent/config/automations.yaml b/release-agent/config/automations.yaml index cadaa09a..a36903d3 100644 --- a/release-agent/config/automations.yaml +++ b/release-agent/config/automations.yaml @@ -50,3 +50,17 @@ automations: - ccd.localization # poll the triggered run (check-localization) purpose: "Poll the localization run every 10 min; email on 3h timeout, post the PR on completion" + # ON-DEMAND (not provisioned at release start). Created only when Build & RC + # Verification is holding on an in-flight re-triggered RC (the human ran + # `rc-retriggered`); torn down when `poll-rc` reports `resolved`. Interval automation, + # so its step needs no fire_at_local. + - slug: build-verify-rc-poller + name: "Release {release} — RC verification poller" + phase: build_verify + every: "30 minutes" + on_demand: true + steps: + - build_verify.rc_report # poll the in-flight RC (poll-rc): re-gate on completion, nudge at 6h + purpose: "Poll a re-triggered RC every 30 min; hold while in-flight, re-apply the gate on completion, courtesy-nudge the owner at 6h" + + diff --git a/release-agent/orchestrator/automations.py b/release-agent/orchestrator/automations.py index bb70d070..72622438 100644 --- a/release-agent/orchestrator/automations.py +++ b/release-agent/orchestrator/automations.py @@ -210,6 +210,10 @@ def plan(config_path: str, release: str, ccd: str) -> dict: "schedule": sched, # one-shot on the CCD date, or an interval poller "one_shot": one_shot, "interval": interval or None, + # ON-DEMAND automations (e.g. the RC poller) are NOT provisioned at release + # start — the skill creates them only when their trigger condition arises + # (an in-flight re-triggered RC) and tears them down when it clears. + "on_demand": bool(d.get("on_demand")), } spec["prompt"] = _prompt_for(spec, release) # Exactly what to record after creating it, so linkage + schedule are captured diff --git a/release-agent/orchestrator/commands/__init__.py b/release-agent/orchestrator/commands/__init__.py index 18d34009..1ce78330 100644 --- a/release-agent/orchestrator/commands/__init__.py +++ b/release-agent/orchestrator/commands/__init__.py @@ -6,7 +6,7 @@ one, so adding a command is a localized change (new/edited module only). """ from . import (release, readiness, pipeline, notify, infra_cmd, automation, - logs, lockdown, notice, step_action, localization, rc_report, sim) + logs, lockdown, notice, step_action, localization, rc_report, rc_poll, sim) # Order controls how subcommands appear in --help. REGISTRARS = [ @@ -19,6 +19,7 @@ notice.register, localization.register, rc_report.register, + rc_poll.register, sim.register, logs.register, automation.register, diff --git a/release-agent/orchestrator/commands/rc_poll.py b/release-agent/orchestrator/commands/rc_poll.py new file mode 100644 index 00000000..124ef6d1 --- /dev/null +++ b/release-agent/orchestrator/commands/rc_poll.py @@ -0,0 +1,129 @@ +"""`poll-rc` — one poll of an in-flight Phase-2 RC verification (the 30-min RC poller). + +After a re-triggered RC (see `rc-retriggered`), the Build & RC Verification phase holds +on an IN-FLIGHT MRWP run (status-aware verify — see steps/build_verify/_common). This +command is the poller seam the `build-verify-rc-poller` automation calls every 30 min: + + 1. advance the drain (`run_until_gate`) so the in-flight verify step re-checks the run's + live status — still running → stays in-flight; completed → the normal stage rule + + UI gate apply and the phase moves on. + 2. emit a deterministic decision the skill acts on: + waiting — still running; nothing to send. + nudge — running past the 6h courtesy threshold; send the owner a heads-up (once). + resolved — the new RC completed and PASSED the gate; Phase 2 advanced (deregister + the poller). + blocked — the new RC completed but re-blocked the gate (still failing). + idle — nothing in-flight (not in Phase 2, or nothing was re-triggered). + +Decisions are pure functions of state; the 6h nudge stamps `nudged_at` on the step so it +is sent at most once. `--now` overrides the clock for the elapsed/nudge math (tests).""" +from __future__ import annotations +import json as _json +from datetime import datetime, timezone + +from orchestrator import cli_common as C + +# The poll cadence + courtesy-nudge threshold. A re-triggered RC that runs longer than +# NUDGE_AFTER_HOURS gets ONE heads-up to the owner (it is not a failure — Scout keeps +# polling), per the agreed Phase-2 blocked-state handling. +POLL_INTERVAL_MIN = 30 +NUDGE_AFTER_HOURS = 6 + +# The verify steps whose run can be in-flight (checker/orchestrator resolve instantly). +_RC_VERIFY_STEPS = ("mrwp_ecs", "mrwp_local") + + +def _parse_iso(s): + try: + return datetime.fromisoformat(str(s).replace("Z", "+00:00")) + except (ValueError, TypeError): + return None + + +def _elapsed_hours(since_iso, now): + since = _parse_iso(since_iso) + if not since: + return 0.0 + if since.tzinfo is None: + since = since.replace(tzinfo=timezone.utc) + return max(0.0, (now - since).total_seconds() / 3600.0) + + +def _nudge_payload(st, sid, hrs: int) -> dict: + """A SHORT courtesy heads-up (not the full RC report) — the re-triggered RC is taking + a while but Scout is still polling; no action needed yet.""" + label = {"mrwp_ecs": "MRWP (ECS)", "mrwp_local": "MRWP (Local)"}.get(sid, sid) + subject = f"[Release {st.release_id}] Re-triggered RC still running after ~{hrs}h" + body = ( + f"Heads-up: the re-triggered {label} run for release {st.release_id} has been " + f"running for about {hrs} hours. This is NOT a failure — Scout is still polling " + f"every {POLL_INTERVAL_MIN} minutes and will re-apply the RC gate the moment the " + f"run completes, with no action needed from you. If a {hrs}h RC run is unexpected, " + f"open the run in ADO to check for a stuck stage.") + teams = (f"⏳ Release {st.release_id}: the re-triggered {label} RC has been running " + f"~{hrs}h. Scout is still polling every {POLL_INTERVAL_MIN}m and re-applies " + f"the gate on completion — no action needed yet.") + return { + "email": {"to": [st.owner_email] if st.owner_email else [], + "subject": subject, "body": body}, + "teams": {"text": teams}, + } + + +def cmd_poll_rc(args): + now = _parse_iso(args.now) if getattr(args, "now", None) else datetime.now(timezone.utc) + if now is None: + print(_json.dumps({"error": f"bad --now: {args.now!r}"})) + return 1 + + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + # Advance: the in-flight verify step re-checks the run's LIVE status. Still running → + # stays in-flight; completed → the stage rule + UI gate run and the phase moves on. + orch.run_until_gate() + st = orch.state + C.save_state(st, args.runs_root, args.release) + + inflight = None + for sid in _RC_VERIFY_STEPS: + s = st.get_step("build_verify", sid) + if s.status == "in_flight": + inflight = (sid, s) + break + + if inflight: + sid, s = inflight + elapsed = _elapsed_hours(s.data.get("in_flight_since"), now) + decision = {"decision": "waiting", "step": sid, + "elapsed_hours": round(elapsed, 2), "poll_in_min": POLL_INTERVAL_MIN} + if elapsed >= NUDGE_AFTER_HOURS and not s.data.get("nudged_at"): + s.data["nudged_at"] = now.isoformat() + st.set_step("build_verify", sid, s) + C.save_state(st, args.runs_root, args.release) + decision["decision"] = "nudge" + decision["nudge"] = _nudge_payload(st, sid, int(elapsed)) + C.emit(args.runs_root, args.release, + f"[rc-poller] {sid} in-flight ~{int(elapsed)}h — 6h courtesy nudge sent " + f"to the owner.", kind="build_verify") + else: + rc = st.get_step("build_verify", "rc_report") + if rc.status == "done": + decision = {"decision": "resolved", "status": "passed", "note": rc.note} + elif rc.status == "blocked": + decision = {"decision": "blocked", "note": rc.note} + else: + decision = {"decision": "idle", + "note": "no in-flight RC in Build & RC Verification"} + + print(_json.dumps(decision)) + return 0 + + +def register(sub): + p = sub.add_parser("poll-rc", + help="One poll of an in-flight Phase-2 RC: advance + emit a " + "waiting/nudge/resolved/blocked/idle decision (30-min poller)") + p.add_argument("--release", required=True) + p.add_argument("--now", default=None, + help="Override 'now' (ISO-8601) for the elapsed / 6h-nudge math") + p.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + p.set_defaults(func=cmd_poll_rc) diff --git a/release-agent/steps/build_verify/rc_report.py b/release-agent/steps/build_verify/rc_report.py index 87252ee9..45bf6e01 100644 --- a/release-agent/steps/build_verify/rc_report.py +++ b/release-agent/steps/build_verify/rc_report.py @@ -82,3 +82,29 @@ def build(state): note=gate["detail"], outbound=True, ) + + +def automation_prompt(release: str, spec: dict) -> str: + """Bespoke instruction for the interval RC poller (owned here, like localization's). + Only the poller shape is used — rc_report has no time-of-day automation.""" + if not spec.get("interval"): + return "" # rc_report is driven by next/tick, not a one-shot automation + return ( + f"Release {release} — RC verification poller (Phase 2, every 30 min).\n" + f"Only act if Build & RC Verification is holding on an IN-FLIGHT re-triggered RC " + f"(the human ran `rc-retriggered`). Poll it once:\n" + f"1. run `poll-rc --release {release}`.\n" + f"2. act on the printed decision:\n" + f" • waiting → still running; send nothing.\n" + f" • nudge → running past 6h; send the courtesy heads-up in decision.nudge " + f"(email decision.nudge.email to the owner AND post decision.nudge.teams.text to " + f"the owner's Scout chat). It is stamped, so it goes out at most once.\n" + f" • resolved → the new RC completed and PASSED the gate; Phase 2 advanced. " + f"Deregister THIS poller (`automation deregister --id `) — " + f"it is no longer needed — and report the pass.\n" + f" • blocked → the new RC completed but re-blocked the UI gate (still failing). " + f"Surface the block to the owner (the 3-exit choice: re-trigger / cherry-pick / " + f"override) and leave the poller in place for the next re-trigger.\n" + f" • idle → nothing in-flight; stay silent.\n" + f"Silently journal: `journal --release {release} --source scout --kind automation " + f"--text \"rc-poller: \"`. Stay silent when there is nothing to send.") diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 36f97203..a740a002 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2561,6 +2561,86 @@ def test_mrwp_run_ids_picks_newest_on_retrigger(): assert ok2 and ids2 == {"ECS": "5", "Local": "6"} +def _run_poll_rc(runs_root, rid, now_iso): + """Run `poll-rc` with the drain stubbed (no live az) and return the decision dict.""" + import io, contextlib, json as _json + from unittest.mock import patch + from orchestrator.commands import rc_poll as RP + + class A: + runs_root = None; release = None; config = CONFIG; as_of = None; now = None + A.runs_root, A.release, A.now = runs_root, rid, now_iso + buf = io.StringIO() + with patch("orchestrator.engine.Orchestrator.run_until_gate", lambda self, **k: []): + with contextlib.redirect_stdout(buf): + rc = RP.cmd_poll_rc(A) + assert rc == 0 + return _json.loads(buf.getvalue().strip().splitlines()[-1]) + + +def test_poll_rc_waits_then_nudges_once_at_6h(): + """The RC poller returns `waiting` while the run is in-flight, sends ONE courtesy + nudge to the owner once it has been in-flight >= 6h, and does not repeat the nudge.""" + import tempfile + from orchestrator.state import StepState + with tempfile.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + st = ReleaseState(release_id=rid, ccd="2026-08-26", ccd_source="confirmed", + owner_email="dev@microsoft.com", owner_name="Dev") + st.set_step("build_verify", "mrwp_ecs", + StepState(status="in_flight", note="RC running", + data={"in_flight_since": "2026-08-20T00:00:00+00:00", + "poll_in_min": 30})) + C.save_state(st, d, rid) + # +2h → still waiting + dec = _run_poll_rc(d, rid, "2026-08-20T02:00:00+00:00") + assert dec["decision"] == "waiting" and dec["step"] == "mrwp_ecs" + assert abs(dec["elapsed_hours"] - 2.0) < 0.01 and dec["poll_in_min"] == 30 + # +7h → nudge (once), addressed to the owner + dec = _run_poll_rc(d, rid, "2026-08-20T07:00:00+00:00") + assert dec["decision"] == "nudge" + assert dec["nudge"]["email"]["to"] == ["dev@microsoft.com"] + assert "polling" in dec["nudge"]["teams"]["text"] + # nudged_at stamped → a later poll is waiting again (no repeat nudge) + dec = _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00") + assert dec["decision"] == "waiting" + + +def test_poll_rc_resolved_blocked_idle(): + """Once no verify step is in-flight, the poller reports the terminal verdict: + resolved (rc_report done) / blocked (rc_report blocked) / idle (nothing running).""" + import tempfile + from orchestrator.state import StepState + with tempfile.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + st = ReleaseState(release_id=rid, ccd="2026-08-26", ccd_source="confirmed") + C.save_state(st, d, rid) + assert _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00")["decision"] == "idle" + + st.set_step("build_verify", "rc_report", StepState(status="done", note="UI CLEAN")) + C.save_state(st, d, rid) + assert _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00")["decision"] == "resolved" + + st.set_step("build_verify", "rc_report", StepState(status="blocked", note="UI 80%")) + C.save_state(st, d, rid) + r = _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00") + assert r["decision"] == "blocked" and r["note"] == "UI 80%" + + +def test_rc_poller_automation_is_on_demand_interval(): + """The build-verify-rc-poller is planned as an on-demand 30-min interval automation + driving rc_report, with a bespoke poll prompt (not the default step-action prompt).""" + from orchestrator import automations as A + assert A.validate(CONFIG) == [] + plan = A.plan(CONFIG, "2026-08", "2026-08-26") + rc = next(a for a in plan["automations"] if a["slug"] == "build-verify-rc-poller") + assert rc["on_demand"] and rc["interval"] == "30 minutes" + assert rc["steps"] == ["build_verify.rc_report"] + assert "poll-rc --release 2026-08" in rc["prompt"] and "6h" in rc["prompt"] + + def test_build_verify_persists_pipeline_run_ids(): """The build_verify steps stash the checker/orchestrator/MRWP runs onto state.pipeline_runs in the nested RC schema, and it round-trips through save/load.""" From b519bd062daf2b43882a6e886419d5c80ca98d04 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:45:17 +0100 Subject: [PATCH 76/90] Phase 2: three-exit blocked message for the <90% RC UI gate The rc_report `attention` block now spells out THREE exits instead of a vague "fix or override", matching the agreed Phase-2 blocked-state handling: 1. Re-trigger (flaky) - re-run the failed RC test run, then `rc-retriggered --release `; Scout tracks the newest RC (holds in-flight, polls 30m, re-gates on completion). 2. Cherry-pick (real bug) - patch via the broker cherry-pick process (link surfaced), then `rc-retriggered` so Scout tracks the fresh RC to completion. 3. Override (LAST RESORT) - `skip ... --reason`, framed explicitly as a team decision to be discussed first (proceeding to Bug Bash with this many UI failures is not a default). Adds CHERRY_PICK_TSG. Updates the skill docs to present all three (never collapse to two): reference/phases/build_verify.md (3-exit flow + in-flight vs blocked), SKILL.md block handling exception, reference/commands.md (rc-retriggered + poll-rc). Strengthens the gate test; suite 196 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/skill/SKILL.md | 2 +- release-agent/skill/reference/commands.md | 2 ++ .../skill/reference/phases/build_verify.md | 31 ++++++++++++++++--- release-agent/steps/build_verify/_common.py | 30 +++++++++++++++--- release-agent/tests/test_engine.py | 4 +++ 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index b0f207a4..050e0db0 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -36,7 +36,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → - **Engine HOLDS at a gate:** present it; `m_ask_user` Approve/Deny; run `approve`/`deny --comment` with their reason; present new status. - **Scout steps pending** (`scout_pending` non-empty in `status`): these are **Scout's automated work, NOT a user to-do** — run them yourself, don't wait for the user and don't present them as "you need to". For EACH id in `scout_pending`: `step-action --release --phase

    --step ` → it returns `needs_skill` (an email/Teams/browser action) → perform the returned `tool`+`payload` (respect any `test_redirect`) → `record-step --step --status pass` (or the step's follow-up, e.g. `check-lockdown`). Do this **silently** for each, then re-run `next`. Only once `scout_pending` is empty do you surface the remaining user holds below. (A scout step that records `attention` becomes a `blocked` user task — handle it like any block.) - **Engine HOLDS for a reminder** (`awaiting_action` with `action`/`needs_owner` — an attest or blocked USER task): present as "you need to do X"; when done, `done --release --note ""`. Not a decision — no Approve/Deny. -- **A step is BLOCKED** (agent found a real problem, e.g. `cg` on High/Critical CG alerts, `cron` on a stale Calendar Checker): show the note plainly. Two exits: **(a) fix** → `next` re-runs the check; **(b) override** → `skip --release --phase

    --step --reason ""`. No other way to clear it. +- **A step is BLOCKED** (agent found a real problem, e.g. `cg` on High/Critical CG alerts, `cron` on a stale Calendar Checker): show the note plainly. Two exits: **(a) fix** → `next` re-runs the check; **(b) override** → `skip --release --phase

    --step --reason ""`. No other way to clear it. **Exception — `build_verify.rc_report` (the <90% UI gate)** has a richer **three-exit** flow: **re-trigger** (flaky → re-run RC, then `rc-retriggered --release `), **cherry-pick** (real bug → patch via the broker cherry-pick process, then `rc-retriggered`), or **override** (`skip …`, the last resort — discuss with the team first). After `rc-retriggered`, Scout tracks the newest RC: the verify step is `in_flight` (⏳ no action) while it runs, and the 30-min poller re-applies the gate on completion. See `reference/phases/build_verify.md`. Present all three — don't collapse it to "fix or override". - **Engine is `scheduled`** (before CCD‑7): relay the opens-date + countdown; nothing to advance. Earlier start = a CCD change (`set-ccd`), not `next`. - **"continue"/"resume":** discover → if gate not cleared show the checklist, else brief with status → `next`. - **User asks ABOUT a step** ("what does X do?", "where do I find the Play Console vitals?", "how do I clear this block?", "why is this needed?", "who fixes this?"): run `step-info --phase

    --step ` and answer from it — do NOT guess step details from memory. It returns the step's what/who/where/how/links/FAQs (accurate, curated in `config/knowledge.yaml`). If it returns "no knowledge entry yet", say so rather than inventing an answer. **Then, if a release is active, silently journal the exchange:** `journal --release --kind qa --phase

    --step --question "" --answer ""` — best-effort, never announced, skip entirely when no release run exists. diff --git a/release-agent/skill/reference/commands.md b/release-agent/skill/reference/commands.md index f7e87faf..65f3c475 100644 --- a/release-agent/skill/reference/commands.md +++ b/release-agent/skill/reference/commands.md @@ -40,6 +40,8 @@ _Loaded on demand. Run all from `C:\repos\android-complete\release-agent`._ | Journal a step Q&A (silent) | `python -m orchestrator.cli journal --release --kind qa --phase

    --step --question "..." --answer "..."` | | Localization: record trigger | `python -m orchestrator.cli record-localization-run --release --build-id ` — store the queued build; leaves the step in-flight | | Localization: one poll | `python -m orchestrator.cli check-localization --release --complete [--logs ""]` — wait / timeout(email) / complete(post PR); acts on the printed decision | +| **Phase 2 — signal a re-triggered RC** | `python -m orchestrator.cli rc-retriggered --release [--reason "..."]` — after the owner re-runs RC (flaky) or the orchestrator triggers a fresh RC (broker cherry-pick), reopens `mrwp_ecs`/`mrwp_local`/`rc_report` so Scout re-evaluates the **newest** RC. Holds `in_flight` (no action) while the run executes; the poller re-applies the gate on completion | +| **Phase 2 — one RC poll** | `python -m orchestrator.cli poll-rc --release ` — advances the drain (in-flight verify re-checks live status) and prints a decision: `waiting` / `nudge` (>6h courtesy heads-up to the owner) / `resolved` (passed → deregister the poller) / `blocked` (re-gate failed) / `idle`. Driven by the on-demand `build-verify-rc-poller` automation (every 30 min) | | Activate conditional hotfix phase | `python -m orchestrator.cli activate --release --phase hotfix` | | **Notify** — push line if something needs me | `python -m orchestrator.cli notify [--release ] [--as-of ] [--force]` | | **Plan timed automations** | `automation plan --release [--json]` — derive the per-release CCD automations (name/schedule/steps/prompt) from `config/automations.yaml` + CCD | diff --git a/release-agent/skill/reference/phases/build_verify.md b/release-agent/skill/reference/phases/build_verify.md index 79e4b22e..19c87a97 100644 --- a/release-agent/skill/reference/phases/build_verify.md +++ b/release-agent/skill/reference/phases/build_verify.md @@ -12,6 +12,12 @@ orchestrator, an auth failure). A blocked step → show the note, then **fix + ` re-check, or **`skip … --reason`** to override. When the chain is green the scout `rc_report` step runs — it is the last Phase-2 step and the go/no-go. +**In-flight vs blocked.** An MRWP step whose RC run is still executing is **`in_flight`** +(⏳ "RC running — Scout is polling"), **not** blocked — a stage that hasn't run *yet* on a +live run is not an aborted pipeline. It needs **no owner action**: the engine holds the +phase and the 30-min poller re-checks until the run completes, then the normal stage rule ++ UI gate apply. Only a stage that never ran on a **completed** run blocks. + ## Automated steps (no skill action — relay from the `status` table) `checker_fired`, `orchestrator_health`, `mrwp_ecs`, `mrwp_local` — read-only `az` agent steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 link. @@ -34,11 +40,26 @@ This is the Phase-2 go/no-go — there is **no separate approval gate**. - **≥ 90% & < 100% → `warn`** — step done; auto-advances into bug bash, but the owner should investigate the failing UI tests **in parallel** (a later step confirms the retest — bug bash is **not** blocked). - - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). Large UI failure: the - owner investigates and decides — patch a real bug + re-trigger RC, or (if it's an - automation flake to re-run later) proceed to bug bash. Exits: fix + re-run, then - `next` re-runs `rc_report`; or `skip … --reason` to override. - - (No UI tests found → `clean` with a ⚠ note.) + - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). This is a large UI + failure. Present the note plainly, then walk the owner through **three exits** (do NOT + reduce it to "fix or override"): + 1. **Re-trigger (flaky)** — if the owner judges the failures are automation flakiness, + they re-run the failed RC test run, then signal **`rc-retriggered --release + --reason "..."`**. That reopens `mrwp_ecs`/`mrwp_local`/`rc_report` so Scout + re-evaluates the **newest** RC. While the new run is still executing the verify step + is **`in_flight`** (⏳ "RC running — Scout is polling") — **no owner action**; the + `build-verify-rc-poller` re-checks every 30 min and re-applies this gate the moment + the run completes. If it runs past 6h the owner gets one courtesy nudge. + 2. **Cherry-pick (real bug)** — if a product bug is driving the failures, the owner + patches it via the **broker cherry-pick process** + (`…/internal-release-checklist/cherry-pick-process-for-broker-libraries`); the + orchestrator then triggers a fresh RC. Same signal: **`rc-retriggered --release + `** so Scout tracks the newest RC to completion. + 3. **Override (LAST RESORT)** — **`skip … --step rc_report --reason ""`**. Frame + this explicitly as the last option: proceeding to Bug Bash with this many UI + failures is a **team decision** and should be **discussed with the team first**, not + taken as a default. The reason is recorded for audit. + (No UI tests found → `clean` with a ⚠ note.) It records the failing-suite summary + stashes the checker/orchestrator/ECS/Local run links on the step. - The command prints `{verdict, blocking, pass_pct, ui_total, detail, links}` for your diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index ba790f51..4a51c796 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -110,6 +110,13 @@ def stash_mrwp(state, provider, snapshot): # (a large UI failure usually means a real regression → fix + re-run MRWP). RC_UI_PASS_THRESHOLD = 90.0 +# The broker-libraries cherry-pick process — the exit for a REAL product bug behind the +# UI failures (patch → orchestrator triggers a fresh RC). Surfaced in the block detail. +CHERRY_PICK_TSG = ("https://eng.ms/docs/microsoft-security/identity/" + "entra-developer-application-platform/auth-client/" + "authn-sdk-msal-android/android-auth-libraries/releases/" + "internal-release-checklist/cherry-pick-process-for-broker-libraries") + # ---------------------------------------------------------------- RC report email def rc_report_model(state, timeout=120): @@ -222,11 +229,24 @@ def rc_ui_gate(model) -> dict: f"failing UI test(s) in parallel (a later step confirms the retest, " f"so bug bash is not blocked)." + _ui_failing_suites_summary(model))} return {**base, "pass_pct": pass_pct, "verdict": "attention", "blocking": True, - "detail": (f"{head} \u2014 BELOW the {thr:.0f}% gate. Large UI failure: investigate " - f"the root cause and decide \u2014 patch a real bug + re-trigger RC, or " - f"(if it's an automation flake to re-run later) proceed to bug bash. This " - f"step stays BLOCKED until you `next` after a re-run, or `skip --reason` " - f"to override." + _ui_failing_suites_summary(model))} + "detail": ( + f"{head} \u2014 BELOW the {thr:.0f}% gate. The RC report was emailed; the " + f"autonomous tick then halted here (it will NOT auto-advance while blocked). " + f"First decide whether this is automation flakiness or a real product bug, " + f"then take ONE of three exits:\n" + f"1) Re-trigger (flaky) \u2014 if these are flaky suites, re-run the failed RC " + f"test run, then signal `rc-retriggered --release --reason \"...\"`. Scout " + f"tracks the NEW RC: it holds (no action) while the run is in-flight, polls " + f"every 30 min, and re-applies this gate the moment it completes.\n" + f"2) Cherry-pick (real bug) \u2014 if a product bug is driving the failures, patch " + f"it via the broker cherry-pick process ({CHERRY_PICK_TSG}); the orchestrator " + f"then triggers a fresh RC. Signal `rc-retriggered --release ` so Scout " + f"tracks the newest RC to completion.\n" + f"3) Override (LAST RESORT) \u2014 `skip --release --phase build_verify " + f"--step rc_report --reason \"\"`. Only after discussing with the team: " + f"proceeding to Bug Bash with this many UI failures is a team decision, not a " + f"default. The reason is recorded for audit." + + _ui_failing_suites_summary(model))} def recovered_unit_tests(model) -> list: diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index a740a002..99acb537 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2204,6 +2204,10 @@ def _model(ecs_ui, local_ui, ecs_suites=None): g2 = K.rc_ui_gate(fail_model) assert g2["verdict"] == "attention" and g2["blocking"] is True and g2["pass_pct"] == 80.0 assert "BELOW" in g2["detail"] and "PROD MSAL - RC Broker (API 32)" in g2["detail"] + # the three exits are spelled out: re-trigger (flaky) / cherry-pick (bug) / override + assert "rc-retriggered" in g2["detail"] + assert "cherry-pick-process-for-broker-libraries" in g2["detail"] + assert "LAST RESORT" in g2["detail"] and "skip" in g2["detail"] # no UI tests anywhere → clean with a warning (absence of data is not a failure) g3 = K.rc_ui_gate({"mrwp": {"ECS": {"tests": {"categories": {}}}, From 32ed96476a698110b503db9eba386d53dcc8bbc9 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 01:20:16 +0100 Subject: [PATCH 77/90] Registry per-release ownership + strip pre-prod legacy/back-compat code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relocate release-owned automations into their release folder and remove dead legacy scaffolding (this project is pre-production — no old on-disk state can exist): Registry ownership: * Release-scoped automations now live in //_automations.json, co-located with that release's release-state.json so ownership is explicit and they're removed with the release folder at close. The machine-wide /_automations.json now holds ONLY shared automations. AutomationRegistry takes an optional release; list/ register/deregister read+write the right file(s). Relocated the existing dev registry. Legacy removals (can't exist without production data): * state.migrate_pipeline_runs (flat->nested) + _versions_str_to_dict helper + its test — the flat pipeline_runs shape only briefly existed earlier in dev before the nested redesign; callers now read state.pipeline_runs directly. * ReleaseState.last_notified ("kept for load compat") — read nowhere. * automation sync's steps-fallback for entries "predating slug" — register always sets slug now, so match by slug only. Kept: ReleaseState.load unknown-key drop (genuine robustness for the unattended automation, not legacy). Docs updated. Suite 196 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/README.md | 21 +++-- .../orchestrator/commands/automation.py | 15 +--- release-agent/orchestrator/registry.py | 88 ++++++++++++++----- release-agent/orchestrator/render.py | 6 +- release-agent/orchestrator/state.py | 67 ++------------ release-agent/steps/build_verify/_common.py | 7 +- release-agent/tests/test_engine.py | 39 ++++---- 7 files changed, 113 insertions(+), 130 deletions(-) diff --git a/release-agent/README.md b/release-agent/README.md index 1101ce30..9a6400e1 100644 --- a/release-agent/README.md +++ b/release-agent/README.md @@ -74,8 +74,9 @@ release-agent/ COMMITTED (distributed with android-complete) .release-runs// GENERATED, gitignored (per-release working state) ├─ release-state.json the per-release metadata + run-state (owner, CCD, steps, gates, …) -└─ events.jsonl the per-release event/interaction log -.release-runs/_automations.json GENERATED, gitignored — registry of provisioned Scout automations +├─ events.jsonl the per-release event/interaction log +└─ _automations.json registry of THIS release's provisioned Scout automations (owned by the release; removed at close) +.release-runs/_automations.json GENERATED, gitignored — registry of SHARED (machine-wide) automations only ``` ## Adding a step (the modular contract) @@ -95,7 +96,7 @@ That's it. Mocking works automatically (`outcome`/knobs); `step-info` shows its guardrail fails loudly if a module and `phases.yaml` drift, so nothing silently breaks. **Two homes for data (by lifetime):** -- **Release metadata + run-state** → `.release-runs//release-state.json` (per-release; the `ReleaseState` record). Holds `owner_email`/`owner_name` (the release owner, resolved from the signed-in `az` user at `init`; reminders email this person), `ccd`/`ccd_source`/`ccd_conflict`, step completion, gate decisions, `last_notified`, etc. Add release-scoped fields here. +- **Release metadata + run-state** → `.release-runs//release-state.json` (per-release; the `ReleaseState` record). Holds `owner_email`/`owner_name` (the release owner, resolved from the signed-in `az` user at `init`; reminders email this person), `ccd`/`ccd_source`/`ccd_conflict`, step completion, gate decisions, `last_notified_date`, etc. Add release-scoped fields here. - **Tool config** → `release-agent/config/*.yaml` (not release-specific; committed): `phases.yaml`, `readiness.yaml`, `schedule.yaml`, `requirements.yaml`. ## Architecture — three layers (so it adapts to other interfaces) @@ -206,12 +207,14 @@ was off is picked up by the next one, and a once-per-calendar-day guard (`last_notified_date`) keeps it to one advance-effect and one email per day. `notify` is the **read-only** variant (report without advancing); `--as-of`/`--force` are debug overrides. -**Automation registry.** Every automation the orchestrator provisions is recorded in -`.release-runs/_automations.json` (via `cli automation register`) so it can be torn -down cleanly. Automations are **per-release** by default (`--release `), created -at start and removed at that release's close (`automation list --release ` → -delete each → `automation deregister`). Push reminders are per-release too. A -`--shared` scope exists for the rare automation meant to outlive every release. +**Automation registry.** Every automation the orchestrator provisions is recorded via +`cli automation register` so it can be torn down cleanly. **Per-release** automations +(`--release `, the default) live in `.release-runs//_automations.json` — +co-located with that release's state so ownership is explicit and they're removed with +the release folder at close (`automation list --release ` → delete each → +`automation deregister`). Push reminders are per-release too. A `--shared` scope (stored +machine-wide at `.release-runs/_automations.json`) exists for the rare automation meant +to outlive every release. ## Two kinds of human step diff --git a/release-agent/orchestrator/commands/automation.py b/release-agent/orchestrator/commands/automation.py index 8ea582a6..58697e5e 100644 --- a/release-agent/orchestrator/commands/automation.py +++ b/release-agent/orchestrator/commands/automation.py @@ -13,7 +13,7 @@ def cmd_automation(args): """Track Scout automations the orchestrator provisions, so they can be torn down at release close. This only records ids + step linkage — the skill does the actual Scout create/delete via m_create_automation / m_delete_automation.""" - reg = AutomationRegistry(args.runs_root) + reg = AutomationRegistry(args.runs_root, getattr(args, "release", None)) if args.action == "plan": return _cmd_plan(args) if args.action == "sync": @@ -108,23 +108,16 @@ def _cmd_sync(args): config_path = getattr(args, "config", None) or C.DEFAULT_CONFIG st = C.load_state(args.runs_root, args.release) ccd = getattr(st, "ccd", None) - reg = AutomationRegistry(args.runs_root) + reg = AutomationRegistry(args.runs_root, getattr(args, "release", None)) registered = reg.list(release=args.release, kind="step-driving") plan = auto_plan.plan(config_path, args.release, ccd) desired_by_slug = {a["slug"]: a for a in plan["automations"]} - desired_by_steps = {tuple(sorted(a["steps"])): a for a in plan["automations"]} updates = [] for e in registered: - # Match by slug (stable, unambiguous). Fall back to steps for older registry - # entries that predate slug — but only when the step set is unique. - spec = desired_by_slug.get(e.get("slug")) - if spec is None and e.get("slug") is None: - key = tuple(sorted(e.get("steps") or [])) - if sum(1 for a in plan["automations"] if tuple(sorted(a["steps"])) == key) == 1: - spec = desired_by_steps.get(key) + spec = desired_by_slug.get(e.get("slug")) # matched by slug (stable, unambiguous) if spec is None: - continue # no unambiguous desired spec — skip + continue # no matching desired spec — skip desired = spec.get("schedule") current = e.get("schedule") updates.append({ diff --git a/release-agent/orchestrator/registry.py b/release-agent/orchestrator/registry.py index 9fc31b1d..b004b84a 100644 --- a/release-agent/orchestrator/registry.py +++ b/release-agent/orchestrator/registry.py @@ -18,10 +18,16 @@ * release-level — operates on the whole release, not a step (e.g. the hourly `tick` push-reminder). Owns NO steps. -Stored machine-wide at /_automations.json (gitignored runtime state). +Storage layout: + * //_automations.json — a release's own automations, co-located + with its release-state.json so ownership is explicit (and they're removed with the + release folder at close). + * /_automations.json — SHARED (machine-wide) automations only; these are + reused across releases and are not tied to any one release folder. """ from __future__ import annotations +import glob import json import os from datetime import datetime, timezone @@ -43,31 +49,51 @@ def _now() -> str: class AutomationRegistry: - def __init__(self, runs_root: str): + def __init__(self, runs_root: str, release: str = None): self.runs_root = runs_root - self.path = os.path.join(runs_root, "_automations.json") + self.release = release + # SHARED (machine-wide) automations only; release automations live in /. + self.shared_path = os.path.join(runs_root, "_automations.json") - def _load(self) -> list: + # ---- paths ---- + def _release_path(self, release: str) -> str: + """A release's own registry file, next to its release-state.json.""" + return os.path.join(self.runs_root, release, "_automations.json") + + def _release_files(self) -> list: + """Every per-release registry file under runs_root.""" + return sorted(glob.glob(os.path.join(self.runs_root, "*", "_automations.json"))) + + # ---- file IO ---- + def _load_file(self, path: str) -> list: try: - with open(self.path, "r", encoding="utf-8") as fh: + with open(path, "r", encoding="utf-8") as fh: data = json.load(fh) return data if isinstance(data, list) else [] except (OSError, ValueError): return [] - def _save(self, entries: list) -> None: - os.makedirs(self.runs_root, exist_ok=True) - tmp = self.path + ".tmp" + def _save_file(self, path: str, entries: list) -> None: + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + tmp = path + ".tmp" with open(tmp, "w", encoding="utf-8") as fh: json.dump(entries, fh, indent=2) - os.replace(tmp, self.path) + os.replace(tmp, path) + + def _path_for(self, entry: dict) -> str: + """Where an entry is stored: its release folder, else the shared file.""" + if entry.get("scope") == "release" and entry.get("release"): + return self._release_path(entry["release"]) + return self.shared_path + # ---- api ---- def register(self, auto_id: str, name: str, release: str = None, shared: bool = False, purpose: str = "", steps: list = None, kind: str = None, schedule: str = None, slug: str = None) -> dict: - """Record an automation (upsert by id). Shared automations store release=None. - `steps` is the list of '.' ids this automation drives — the - automation<->step linkage used for traceability. + """Record an automation (upsert by id). Shared automations store release=None and + live in the machine-wide file; release automations live in /. `steps` is + the list of '.' ids this automation drives — the automation<->step + linkage used for traceability. `slug` is the stable identity from config/automations.yaml (e.g. 'ccd-noon'). It's the reliable key for `automation sync` — matching by steps alone is @@ -94,30 +120,48 @@ def register(self, auto_id: str, name: str, release: str = None, "slug": slug or None, "kind": kind, "scope": "shared" if shared else "release", - "release": None if shared else release, + "release": None if shared else (release or self.release), "purpose": purpose, "steps": steps, "schedule": schedule or None, "registered_at": _now(), } - entries = [e for e in self._load() if e.get("id") != auto_id] # upsert + # Upsert: drop any prior copy of this id wherever it lived, then write to its + # (possibly new) home file. + self._remove_everywhere(auto_id) + path = self._path_for(entry) + entries = self._load_file(path) entries.append(entry) - self._save(entries) + self._save_file(path, entries) return entry + def _remove_everywhere(self, auto_id: str) -> bool: + """Drop `auto_id` from the shared file and every per-release file. Returns True + if it was found somewhere.""" + removed = False + for path in [self.shared_path, *self._release_files()]: + entries = self._load_file(path) + kept = [e for e in entries if e.get("id") != auto_id] + if len(kept) != len(entries): + self._save_file(path, kept) + removed = True + return removed + def deregister(self, auto_id: str) -> bool: - entries = self._load() - kept = [e for e in entries if e.get("id") != auto_id] - self._save(kept) - return len(kept) != len(entries) + return self._remove_everywhere(auto_id) def list(self, release: str = None, scope: str = None, step: str = None, kind: str = None) -> list: - """List entries. `release` filters to that release's automations (scope - 'release' whose release matches). `scope` filters by 'shared'/'release'. + """List entries. `release` filters to that release's automations (and reads only + that release's file + the shared file). `scope` filters by 'shared'/'release'. `step` filters to automations that DRIVE that '.' id (reverse lookup). `kind` filters by 'step-driving'/'release-level'.""" - entries = self._load() + entries = self._load_file(self.shared_path) + if release is not None: + entries += self._load_file(self._release_path(release)) + else: + for path in self._release_files(): + entries += self._load_file(path) if release is not None: entries = [e for e in entries if e.get("release") == release] if scope is not None: diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 88558c74..3733cc4f 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -247,11 +247,9 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: def _pipelines_line(r: dict) -> str: """Compact one-line summary of the Phase-2 release-pipeline runs recorded on state (checker → orchestrator → the LATEST RC's two MRWP runs). Empty string when none - resolved yet. Reads the nested pipeline_runs schema (migrating a legacy flat shape); - no live az call in the render path.""" - from orchestrator.state import migrate_pipeline_runs + resolved yet. Reads the nested pipeline_runs schema; no live az call in the render path.""" from tools.pipelines import format_versions - pr = migrate_pipeline_runs(r.get("pipeline_runs") or {}) + pr = r.get("pipeline_runs") or {} parts = [] ch = pr.get("checker") or {} if ch.get("run_id"): diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py index 513da2d5..9b6feb54 100644 --- a/release-agent/orchestrator/state.py +++ b/release-agent/orchestrator/state.py @@ -23,54 +23,6 @@ def _now() -> str: return datetime.now(timezone.utc).isoformat() -def _versions_str_to_dict(v) -> dict: - """'Common 24.6.0, Msal 8.4.2, Broker 16.5.0' -> {Common,Msal,Broker}. A dict passes - through; anything unparseable yields {}.""" - if isinstance(v, dict): - return {k: val for k, val in v.items() if val} - if not isinstance(v, str) or not v.strip(): - return {} - out = {} - for part in v.split(","): - toks = part.strip().split() - if len(toks) >= 2: - out[toks[0]] = toks[1] - return out - - -def migrate_pipeline_runs(pr) -> dict: - """Normalize the pipeline_runs container to the nested RC schema (idempotent). - - Accepts the legacy FLAT shape - {checker, orchestrator, versions, mrwp_ecs, mrwp_local, mrwp_id_source, resolved_at} - and lifts it to - {checker:{run_id,...}, orchestrator:{run_id,versions:{},...}, rcs:[{rc:1,ecs,local}]}. - A value already in the nested shape (has 'rcs', or a dict 'checker') is returned as-is. - Empty/None -> {}.""" - if not pr or not isinstance(pr, dict): - return {} - # Already nested? (rcs present, or checker is an object) - if "rcs" in pr or isinstance(pr.get("checker"), dict) or isinstance(pr.get("orchestrator"), dict): - return pr - out = {} - if pr.get("checker"): - out["checker"] = {"run_id": str(pr["checker"]), "resolved_at": pr.get("resolved_at")} - if pr.get("orchestrator"): - out["orchestrator"] = {"run_id": str(pr["orchestrator"]), - "versions": _versions_str_to_dict(pr.get("versions")), - "resolved_at": pr.get("resolved_at")} - ecs, local = pr.get("mrwp_ecs"), pr.get("mrwp_local") - if ecs or local: - rc = {"rc": 1, "resolved_at": pr.get("resolved_at")} - src = pr.get("mrwp_id_source") - if ecs: - rc["ecs"] = {"run_id": str(ecs), "id_source": src} - if local: - rc["local"] = {"run_id": str(local), "id_source": src} - out["rcs"] = [rc] - return out - - @dataclass class StepState: """Persisted state for a single step.""" @@ -130,14 +82,11 @@ class ReleaseState: steps: dict = field(default_factory=dict) # "phase.step" -> StepState (as dict) gate_decisions: list = field(default_factory=list) pending_human: list = field(default_factory=list) # outstanding human actions - last_notified: Optional[str] = None # last push message (legacy; kept for load compat) last_notified_date: Optional[str] = None # YYYY-MM-DD of the last daily digest sent - # Phase-2 release-pipeline run ids (checker / orchestrator / the two MRWP runs), - # refreshed each time Phase 2 resolves the chain. Because a re-triggered 'Trigger RC - # Testing' stage spawns NEW MRWP runs, these are re-resolved (newest wins) — not a - # fixed cache. Surfaced in status details + the daily digest. # Phase-2 release-pipeline runs — the RECORD of what verification resolved, reused by - # the RC report + gate (no re-discovery). Nested schema (see migrate_pipeline_runs): + # the RC report + gate (no re-discovery). Because a re-triggered 'Trigger RC Testing' + # stage spawns NEW MRWP runs, these are re-resolved (newest wins) — not a fixed cache. + # Nested schema: # { checker: {run_id, when, resolved_at}, # orchestrator: {run_id, versions:{Common,Msal,Broker}, parked, resolved_at}, # rcs: [ {rc, ecs:{run_id,id_source,complete,ran,total,failed_stages, @@ -154,15 +103,11 @@ class ReleaseState: def load(cls, path: str) -> "ReleaseState": with open(path, "r", encoding="utf-8") as fh: data = json.load(fh) - # Tolerate unknown/legacy keys: a persisted state file may predate a - # field rename/removal (or be hand-edited), and this loader runs in an - # unattended automation — an unexpected key must never hard-crash it. - # Only keys matching a declared field are applied; the rest are dropped. + # Drop unknown keys: this loader runs in an unattended automation, so a + # hand-edited or forward-version state file must never hard-crash it. Only keys + # matching a declared field are applied. known = {f.name for f in fields(cls)} obj = cls(**{k: v for k, v in data.items() if k in known}) - # Migrate the flat pre-nested pipeline_runs shape to the nested RC schema so - # older/hand-edited state files load cleanly and readers see one shape. - obj.pipeline_runs = migrate_pipeline_runs(obj.pipeline_runs) return obj def save(self, path: str) -> None: diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 4a51c796..cc2c99d4 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -19,7 +19,6 @@ CHECKER_DEF, ORCHESTRATOR_DEF, MRWP_DEF, ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, format_versions, ) -from orchestrator.state import migrate_pipeline_runs from orchestrator.outcomes import Done, Blocked, InProgress from steps.lib.mockctx import mock_input, MISSING @@ -51,8 +50,8 @@ def _now_iso(): def _pipeline_runs(state) -> dict: - """The nested pipeline_runs container on state (migrating a legacy flat shape).""" - return migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + """The nested pipeline_runs container on state.""" + return getattr(state, "pipeline_runs", None) or {} def stash_checker(state, run_id, when=None): @@ -125,7 +124,7 @@ def rc_report_model(state, timeout=120): iteration (rcs[-1]) and routes through `tools.pipelines.assemble_rc_model` — the SAME assembler the live path uses — so the state-based model can't drift from the live one. """ - pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + pr = getattr(state, "pipeline_runs", None) or {} ch = pr.get("checker") or {} o = pr.get("orchestrator") or {} rcs = pr.get("rcs") or [] diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 99acb537..24203e24 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -1378,6 +1378,26 @@ def test_registry_register_list_deregister(): assert len(reg.list()) == 1 +def test_registry_relocates_release_automations_into_release_folder(): + """Release-scoped automations live in //_automations.json (owned + by the release); shared ones stay machine-wide.""" + from orchestrator.registry import AutomationRegistry + import os as _os, json as _json + with tempfile.TemporaryDirectory() as tmp: + reg = AutomationRegistry(tmp, release="2026-08") + reg.register("a2", "Phase-3 watcher", release="2026-08", steps=["bug_bash.bash_done"]) + reg.register("sh", "Release push reminders", shared=True, purpose="push") + rel_file = _os.path.join(tmp, "2026-08", "_automations.json") + shared_file = _os.path.join(tmp, "_automations.json") + # the release automation is co-located with the release; shared stays machine-wide + assert [e["id"] for e in _json.load(open(rel_file))] == ["a2"] + assert [e["id"] for e in _json.load(open(shared_file))] == ["sh"] + # release listing reads the release file + shared; deregister finds it in-folder + assert {e["id"] for e in reg.list(release="2026-08")} == {"a2"} + assert reg.deregister("a2") is True + assert reg.list(release="2026-08") == [] + + def test_registry_records_step_linkage_and_reverse_lookup(): """An automation entry records the steps it drives + its kind; list(step=...) is the reverse lookup (which automation owns a step) — the traceability link.""" @@ -2667,25 +2687,6 @@ def test_build_verify_persists_pipeline_run_ids(): assert again.pipeline_runs["rcs"][-1]["ecs"]["run_id"] == "900001" -def test_migrate_pipeline_runs_flat_to_nested(): - """A legacy FLAT pipeline_runs shape migrates to the nested RC schema on load - (idempotent); an already-nested value passes through unchanged.""" - from orchestrator.state import migrate_pipeline_runs as M - flat = {"checker": "111", "orchestrator": "222", - "versions": "Common 24.6.0, Msal 8.4.2, Broker 16.5.0", - "mrwp_ecs": "333", "mrwp_local": "444", "mrwp_id_source": "tags", - "resolved_at": "2026-08-20T00:00:00Z"} - m = M(flat) - assert m["checker"]["run_id"] == "111" - assert m["orchestrator"]["run_id"] == "222" - assert m["orchestrator"]["versions"] == {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"} - assert m["rcs"] == [{"rc": 1, "resolved_at": "2026-08-20T00:00:00Z", - "ecs": {"run_id": "333", "id_source": "tags"}, - "local": {"run_id": "444", "id_source": "tags"}}] - assert M(m) == m # idempotent - assert M({}) == {} - - def test_stash_mrwp_appends_new_rc_on_id_change(): """stash_mrwp merges ecs+local into ONE rc entry, and appends a NEW rc iteration only when a provider's run id changes (RC Testing re-triggered). Latest = rcs[-1].""" From 02a86f4705e064f3231178cd96adcc59f3463d09 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 01:48:21 +0100 Subject: [PATCH 78/90] Phase 3: implement clone_plans as two real agents (Broker + Authenticator) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the single clone_plans stub with two real ADO-backed agent steps, per the distinct team procedures: * clone_plans_broker — COPIES the Broker master test plan (#2007357, area Engineering\Auth Client\Broker\Android) to a new plan "Android Monthly Release - ", referencing existing test cases (ADO clone default). Follows the eng.ms broker test-plans doc. * clone_plans_auth — CREATES a query-based (dynamic) test suite under "MSAuthenticator Test Passes" (#714514/714515) named "Android/release/MM/YYYY", WIQL mirroring the live suite "Android/release/08/2024" (tag Android, Identity Apps area, not Closed/IgnoreOnPrem). Follows the IDWiki 33580 doc and STOPS before assigning testers (later step). Both are idempotent: the created plan/suite id is stashed on the step and re-confirmed on re-run (auth also reuses a same-named suite) so a re-entered Phase 3 never makes duplicates. New tools/testplans.py wraps the ADO test-plan clone + dynamic-suite-create APIs; adds a pipelines._ado_rest_send POST primitive (the first ADO write path). Query/name shapes and plan ids were verified live read-only against the real plans. config/phases.yaml splits the step; knowledge.yaml documents both for step-info; tests cover clone/idempotency/block offline via mock knobs (net-guarded). Suite 202 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 50 +++++ release-agent/config/phases.yaml | 3 +- release-agent/mocks.local.example.yaml | 3 +- release-agent/steps/bug_bash/__init__.py | 1 + .../steps/bug_bash/clone_plans_auth.py | 101 ++++++++++ .../steps/bug_bash/clone_plans_broker.py | 80 ++++++++ release-agent/tests/test_engine.py | 77 +++++++- release-agent/tools/pipelines.py | 43 +++++ release-agent/tools/testplans.py | 180 ++++++++++++++++++ 9 files changed, 532 insertions(+), 6 deletions(-) create mode 100644 release-agent/steps/bug_bash/__init__.py create mode 100644 release-agent/steps/bug_bash/clone_plans_auth.py create mode 100644 release-agent/steps/bug_bash/clone_plans_broker.py create mode 100644 release-agent/tools/testplans.py diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 7aceb7db..c91ac51e 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -336,6 +336,56 @@ build_verify.rc_report: - q: "Does the sim send this email?" a: "No. In a sim the step is marked done without sending; only the real skill flow composes + sends it via WorkIQ." +# =============================== Phase 3 · bug_bash =============================== + +bug_bash.clone_plans_broker: + summary: "Copy the Broker master test plan to a new plan for this release." + what: > + Each release COPIES the Broker master template (ADO test plan #2007357, area + 'Engineering\\Auth Client\\Broker\\Android') to a new plan named + 'Android Monthly Release - ' (e.g. 'Android Monthly Release - Aug 2026'), + REFERENCING the existing test cases (the ADO clone default — it shares test cases, + it does not duplicate them). This is the Broker half of the old clone_plans step. + who: "Scout runs it automatically during `next` (an agent step, via the ADO test-plan clone API)." + how: > + Idempotent — the new plan id is stashed on the step; a re-run re-confirms that plan + exists and reports done without cloning again. If the clone fails (auth/API), the + step blocks with the ADO error; fix and `next`, or `skip --reason` to override. + links: + - name: "Broker master test plan (#2007357)" + url: "https://identitydivision.visualstudio.com/Engineering/_testPlans/define?planId=2007357&suiteId=2008656" + - name: "Broker test-plans doc" + url: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/internal-release-checklist/test-plans" + faqs: + - q: "Does it duplicate the test cases?" + a: "No — it references the existing test cases (the 'Reference existing test cases' clone option). Edit test cases only in the master template." + +bug_bash.clone_plans_auth: + summary: "Create the Authenticator bug-bash query-based test suite for this release." + what: > + Creates a NEW query-based (dynamic) test suite under the standing 'MSAuthenticator + Test Passes' plan (#714514, root suite 714515), named 'Android/release/MM/YYYY' + (e.g. 'Android/release/08/2026'). Its WIQL selects the Android bug-bash test cases + (tag 'Android', area 'Engineering\\ISP\\Identity Apps', not Closed, not 'IgnoreOnPrem' + — mirrors the live suite 'Android/release/08/2024'). This is the Authenticator half + of the old clone_plans step. It STOPS after creating the suite — assigning testers is + a later, manual step. + who: "Scout runs it automatically during `next` (an agent step, via the ADO test-suite create API)." + how: > + Idempotent — the created suite id is stashed on the step, and it also reuses an + existing same-named suite rather than making a duplicate. If the create fails + (auth/API), the step blocks with the ADO error; fix and `next`, or `skip --reason`. + links: + - name: "MSAuthenticator Test Passes plan (#714514)" + url: "https://identitydivision.visualstudio.com/Engineering/_testManagement?_a=tests&planId=714514&suiteId=714515" + - name: "How to make a test suite for bug bash (IDWiki 33580)" + url: "https://identitydivision.visualstudio.com/IdentityWiki/_wiki/wikis/IdentityWiki.wiki/33580/How-to-make-test-suite-for-bug-bash-" + faqs: + - q: "Does it assign testers?" + a: "No — it stops after creating the query-based suite. Assigning testers is done later (a separate, manual step)." + - q: "What names the suite?" + a: "The release: 'Android/release/MM/YYYY' — mirroring the live convention (newest was 'Android/release/08/2024')." + # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the # step entries (summary/what/who/where/how/links/faqs). diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 41e4d664..96675dd9 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -63,7 +63,8 @@ phases: name: "Test / Bug Bash" checklist_phase: 3 steps: - - { id: clone_plans, name: "Clone/rename test plans", owner: agent, maps_to: [T3] } + - { id: clone_plans_broker, name: "Clone Broker test plan (copy master template)", owner: agent, maps_to: [T3] } + - { id: clone_plans_auth, name: "Create Authenticator bug-bash test suite", owner: agent, maps_to: [T3] } - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", owner: agent, maps_to: [T2] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", owner: agent, maps_to: [T5] } diff --git a/release-agent/mocks.local.example.yaml b/release-agent/mocks.local.example.yaml index 7929d6b9..0c195e0e 100644 --- a/release-agent/mocks.local.example.yaml +++ b/release-agent/mocks.local.example.yaml @@ -84,7 +84,8 @@ preflight.cg: # build_verify.rc_report: { outcome: done } # scout (emails RC report + applies the 90% UI gate) # ---- Phase 3 · bug_bash ---- -# bug_bash.clone_plans: { outcome: done } # agent +# bug_bash.clone_plans_broker: { outcome: done } # agent (copy Broker master plan) +# bug_bash.clone_plans_auth: { outcome: done } # agent (create Authenticator query-suite) # bug_bash.coordinate: { outcome: done } # agent # bug_bash.ui_failures: { outcome: done } # reminder (human) # bug_bash.signoffs: { outcome: done } # agent diff --git a/release-agent/steps/bug_bash/__init__.py b/release-agent/steps/bug_bash/__init__.py new file mode 100644 index 00000000..29013d34 --- /dev/null +++ b/release-agent/steps/bug_bash/__init__.py @@ -0,0 +1 @@ +"""Phase 3 (bug_bash) step modules.""" diff --git a/release-agent/steps/bug_bash/clone_plans_auth.py b/release-agent/steps/bug_bash/clone_plans_auth.py new file mode 100644 index 00000000..10582d48 --- /dev/null +++ b/release-agent/steps/bug_bash/clone_plans_auth.py @@ -0,0 +1,101 @@ +"""Step: `clone_plans_auth` — create the Authenticator bug-bash test suite for this +release (Phase 3, bug_bash). + +Per the Authenticator "How to make a test suite for bug bash" doc, each release creates +a NEW query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" +plan (714514 / rootSuite 714515), named after the release ("Android/release/MM/YYYY"), +whose WIQL selects the Android bug-bash test cases. This is the Authenticator half of the +old `clone_plans` stub. + +We CREATE the suite and STOP — assigning testers is a later, manual step (out of scope +here, per the doc's "Assign Testers" cut line). + +Idempotent: the created suite id is stashed on the step (data.suite_id); a re-run +re-confirms it and reports done without creating a duplicate. As a second guard the step +also looks for an existing same-named child suite before creating. + +Mock knobs (mocks.local.yaml / tests): + suite_id : pretend the suite already exists (this id) — verifies + reports done. + create_id : the id the create should "return" (skip the live create). + existing : id of an already-present same-named suite the name-scan should "find". + fail : a detail string → force a Blocked (simulate an API/auth failure). +""" +from __future__ import annotations + +from orchestrator.outcomes import Done, Blocked +from steps.lib.agent import legacy_run +from steps.lib.mockctx import mock_input, MISSING +from tools import testplans as T + +ID = "clone_plans_auth" +KIND = "agent" + +MOCKABLE = { + "suite_id": {"kind": "input", "desc": "Pretend the query-suite already exists (this id)."}, + "create_id": {"kind": "input", "desc": "Id the create should return (skip the live create)."}, + "existing": {"kind": "input", "desc": "Id an existing same-named suite the name-scan finds."}, + "fail": {"kind": "input", "desc": "Force a Blocked with this detail (simulate an API failure)."}, +} + + +def _links(suite_id): + return [{"name": f"Authenticator bug-bash suite {suite_id}", + "url": T.plan_web_url(T.AUTH_PLAN, suite_id)}] + + +def build(state): + fail = mock_input("fail", MISSING) + if fail is not MISSING: + return Blocked(f"clone_plans_auth: {fail}") + + name = T.auth_suite_name(state.release_id) + step = state.get_step("bug_bash", ID) + + # Already created? (test-injected id, or a stored id from a prior run) → done. + injected = mock_input("suite_id", MISSING) + if injected is not MISSING: + return Done(f"Authenticator bug-bash suite already exists for {state.release_id}: " + f"'{name}' (suite {injected}).", links=_links(injected)) + stored = (step.data or {}).get("suite_id") + if stored: + ok, info, _ = T.get_suite(T.AUTH_PLAN, stored) + if ok and info: + return Done(f"Authenticator bug-bash suite already exists for {state.release_id}: " + f"'{info.get('name') or name}' (suite {stored}).", links=_links(stored)) + # recorded id no longer resolves — fall through and re-create + + # Duplicate guard: is a same-named suite already under the root? (offline-injectable) + existing = mock_input("existing", MISSING) + if existing is MISSING: + ok, existing, detail = T.find_child_suite_by_name(T.AUTH_PLAN, T.AUTH_ROOT_SUITE, name) + if not ok: + hint = " — run `az login`" if str(detail).startswith("AUTH") else "" + return Blocked(f"clone_plans_auth: could not list suites under the " + f"MSAuthenticator plan (#{T.AUTH_PLAN}) ({detail}){hint}.") + if existing: + step.data = dict(step.data or {}); step.data["suite_id"] = existing + state.set_step("bug_bash", ID, step) + return Done(f"Authenticator bug-bash suite already exists for {state.release_id}: " + f"'{name}' (suite {existing}).", links=_links(existing)) + + # Create the query-based suite. + create_id = mock_input("create_id", MISSING) + if create_id is MISSING: + ok, create_id, detail = T.create_auth_query_suite(name, T.auth_bugbash_query()) + if not ok: + hint = " — run `az login`" if str(detail).startswith("AUTH") else "" + return Blocked( + f"clone_plans_auth: could not create the query-based suite '{name}' under " + f"the MSAuthenticator plan (#{T.AUTH_PLAN}) ({detail}){hint}.") + + step.data = dict(step.data or {}) + step.data["suite_id"] = create_id + step.data["suite_name"] = name + state.set_step("bug_bash", ID, step) + return Done( + f"Created the Authenticator bug-bash query-suite '{name}' (suite {create_id}) " + f"under 'MSAuthenticator Test Passes'. Next: assign testers (later step).", + links=_links(create_id)) + + +run = legacy_run(build) diff --git a/release-agent/steps/bug_bash/clone_plans_broker.py b/release-agent/steps/bug_bash/clone_plans_broker.py new file mode 100644 index 00000000..5ca9b0eb --- /dev/null +++ b/release-agent/steps/bug_bash/clone_plans_broker.py @@ -0,0 +1,80 @@ +"""Step: `clone_plans_broker` — copy the Broker master test plan for this release +(Phase 3, bug_bash). + +Per the broker release doc, each release COPIES the master template plan (2007357) +to a new plan "Android Monthly Release - ", referencing the existing test +cases (the ADO clone default — shares test cases, doesn't duplicate them). This is the +Broker half of the old `clone_plans` stub. + +Idempotent: the created plan id is stashed on the step (data.plan_id). On a re-run the +step re-confirms that plan still exists and reports done WITHOUT cloning again — so a +`next` that re-enters Phase 3 never spawns a duplicate plan. + +Mock knobs (mocks.local.yaml / tests): + plan_id : pretend the clone already ran (this plan id) — verifies + reports done. + clone_id : the id the clone POST should "return" (skip the live CloneOperation). + fail : a detail string → force a Blocked (simulate an API/auth failure). +""" +from __future__ import annotations + +from orchestrator.outcomes import Done, Blocked +from steps.lib.agent import legacy_run +from steps.lib.mockctx import mock_input, MISSING +from tools import testplans as T + +ID = "clone_plans_broker" +KIND = "agent" + +MOCKABLE = { + "plan_id": {"kind": "input", "desc": "Pretend the clone already produced this plan id (idempotency test)."}, + "clone_id": {"kind": "input", "desc": "Id the clone should return (skip the live CloneOperation)."}, + "fail": {"kind": "input", "desc": "Force a Blocked with this detail (simulate an API failure)."}, +} + + +def _links(plan_id): + return [{"name": f"Broker test plan {plan_id}", "url": T.plan_web_url(plan_id)}] + + +def build(state): + fail = mock_input("fail", MISSING) + if fail is not MISSING: + return Blocked(f"clone_plans_broker: {fail}") + + dest = T.broker_plan_name(state.release_id) + step = state.get_step("bug_bash", ID) + + # Already cloned? A test injects `plan_id` to assert idempotency; otherwise the + # stored id from a prior run is re-confirmed against ADO. Either way → done, no re-clone. + injected = mock_input("plan_id", MISSING) + if injected is not MISSING: + return Done(f"Broker test plan already cloned for {state.release_id}: " + f"'{dest}' (plan {injected}).", links=_links(injected)) + stored = (step.data or {}).get("plan_id") + if stored: + ok, info, _ = T.get_plan(stored) + if ok and info: + return Done(f"Broker test plan already cloned for {state.release_id}: " + f"'{info.get('name') or dest}' (plan {stored}).", links=_links(stored)) + # recorded id no longer resolves — fall through and re-clone + + # Clone the master (or take the injected clone_id offline). + clone_id = mock_input("clone_id", MISSING) + if clone_id is MISSING: + ok, clone_id, detail = T.clone_broker_plan(dest) + if not ok: + hint = " — run `az login`" if str(detail).startswith("AUTH") else "" + return Blocked( + f"clone_plans_broker: could not copy the master test plan " + f"(#{T.BROKER_MASTER_PLAN}) to '{dest}' ({detail}){hint}.") + + step.data = dict(step.data or {}) + step.data["plan_id"] = clone_id + step.data["plan_name"] = dest + state.set_step("bug_bash", ID, step) + return Done( + f"Copied the Broker master test plan (#{T.BROKER_MASTER_PLAN}) to '{dest}' " + f"(plan {clone_id}), referencing existing test cases.", links=_links(clone_id)) + + +run = legacy_run(build) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 24203e24..19460240 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -72,6 +72,9 @@ def _recent_iso(): {"name": "UI Automation", "state": "completed", "result": "failed"}], "tests": {"total": 100, "passed": 98, "failed": 2}}, "build_verify.rc_report": {"outcome": "done", "note": "RC report emailed (test)"}, # skip live az + send + # Phase-3 bug_bash clone steps — real agents; keep flow tests offline. + "bug_bash.clone_plans_broker": {"outcome": "done", "note": "broker plan cloned (test)"}, + "bug_bash.clone_plans_auth": {"outcome": "done", "note": "auth suite created (test)"}, } @@ -635,9 +638,9 @@ def test_holds_at_first_hold(): assert actions[-1].step == "ui_failures" # Phases 0-2 gateless (rc_report auto); first hold is Phase-3 ui_failures # auto steps that RUN before the first hold: Phase-0 breaking/cg/cron/wiki (4) + # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout - # email, mocked done here) (1) + Phase-3 clone_plans/coordinate stubs (2). The Phase-1 - # scout steps are pre-recorded by _orch's _clear_ccd_scout (not "ran"). - assert sum(1 for a in actions if a.kind == "ran") == 11 + # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/coordinate + # stubs (3). The Phase-1 scout steps are pre-recorded by _orch's _clear_ccd_scout (not "ran"). + assert sum(1 for a in actions if a.kind == "ran") == 12 def test_gate_blocks_until_approved(): @@ -1015,7 +1018,7 @@ def test_only_frontier_phase_shows_current_despite_stale_downstream_progress(): for pid in ("preflight", "ccd"): for s in next(p for p in orch.config["phases"] if p["id"] == pid)["steps"]: orch.state.set_step(pid, s["id"], StepState(status="done", by="test")) - for sid in ("clone_plans", "coordinate"): + for sid in ("clone_plans_broker", "coordinate"): orch.state.set_step("bug_bash", sid, StepState(status="done", by="test")) phases = {p["id"]: p for p in orch.status_report()["phases"]} assert phases["build_verify"]["state"] == "current" and phases["build_verify"]["current"] @@ -2708,6 +2711,72 @@ def test_stash_mrwp_appends_new_rc_on_id_change(): assert K.latest_rc(st)["ecs"]["run_id"] == "910001" # latest = last +# ---- Phase 3: clone_plans_broker / clone_plans_auth ---- + +def _bb_build(sid, mocks, release="2026-08"): + """Build a bug_bash step outcome offline with the given mock inputs active.""" + import steps as _steps + from steps.lib import mockctx + from orchestrator.outcomes import as_dict + st = ReleaseState(release_id=release) + with mockctx.active(mocks): + return st, as_dict(_steps.get_step("bug_bash", sid).build(st)) + + +def test_testplans_names_and_query(): + from tools import testplans as T + assert T.broker_plan_name("2026-08") == "Android Monthly Release - Aug 2026" + assert T.auth_suite_name("2026-08") == "Android/release/08/2026" + q = T.auth_bugbash_query() + assert "contains 'Android'" in q and "IgnoreOnPrem" in q and "Identity Apps" in q + + +def test_clone_plans_broker_clones_then_idempotent(): + """First run copies the master plan and stashes the new plan id; a re-run with that id + stored reports done WITHOUT cloning again.""" + st, out = _bb_build("clone_plans_broker", {"clone_id": "5551212"}) + assert out["kind"] == "done" + assert "Copied the Broker master test plan" in out["note"] and "5551212" in out["note"] + assert st.get_step("bug_bash", "clone_plans_broker").data["plan_id"] == "5551212" + assert out["links"][0]["url"].endswith("planId=5551212") + # idempotent: an injected existing plan id → already-cloned, no re-clone + _, out2 = _bb_build("clone_plans_broker", {"plan_id": "5551212"}) + assert out2["kind"] == "done" and "already cloned" in out2["note"] + + +def test_clone_plans_broker_blocks_on_api_failure(): + _, out = _bb_build("clone_plans_broker", {"fail": "HTTP 403: forbidden"}) + assert out["kind"] == "blocked" and "403" in out["reason"] + + +def test_clone_plans_auth_creates_query_suite_then_idempotent(): + """First run creates the query-based suite (name + stash); a re-run with the suite id + stored reports done without re-creating.""" + st, out = _bb_build("clone_plans_auth", {"existing": None, "create_id": "778899"}) + assert out["kind"] == "done" + assert "Created the Authenticator bug-bash query-suite 'Android/release/08/2026'" in out["note"] + assert "assign testers" in out["note"].lower() # stops before assigning testers + assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "778899" + # idempotent via injected existing suite id + _, out2 = _bb_build("clone_plans_auth", {"suite_id": "778899"}) + assert out2["kind"] == "done" and "already exists" in out2["note"] + + +def test_clone_plans_auth_reuses_existing_same_named_suite(): + """If a same-named suite already exists under the root, reuse it (no duplicate create).""" + st, out = _bb_build("clone_plans_auth", {"existing": "424242"}) + assert out["kind"] == "done" and "already exists" in out["note"] + assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "424242" + + +def test_bug_bash_clone_steps_are_real_agents(): + """Both clone steps resolve to real agent modules (KIND=agent) — no longer stubs.""" + import steps as _steps + for sid in ("clone_plans_broker", "clone_plans_auth"): + mod = _steps.get_step("bug_bash", sid) + assert mod is not None and getattr(mod, "KIND", None) == "agent" and hasattr(mod, "run") + + def test_digest_shows_rc_line_when_build_verify_active(): """When a phase that opts in (show_pipeline_runs) is active and run ids are on state, the daily digest carries a one-line RC summary; phases that don't opt in omit it.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 7853c9be..75fbd3a7 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -121,6 +121,49 @@ def _ado_rest_get_text(url, timeout): return (False, None, f"REST GET failed: {e}") +def _ado_rest_send(url, method, body, timeout): + """Send a JSON REST request (POST/PATCH/PUT) to ADO with an az-minted bearer token. + Returns (ok, json, detail). The ONE write primitive — used for test-plan creates + where `az devops invoke` has no clean surface.""" + az = shutil.which("az") + if az is None: + return (False, None, "az CLI not found") + try: + tok = subprocess.run( + [az, "account", "get-access-token", "--resource", _ADO_RESOURCE, + "--query", "accessToken", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8") + except (subprocess.TimeoutExpired, OSError) as e: + return (False, None, f"failed to get token: {e}") + if tok.returncode != 0 or not (tok.stdout or "").strip(): + return (False, None, "AUTH: could not get an ADO token (run `az login`)") + token = tok.stdout.strip() + import urllib.request + import urllib.error + data = _json.dumps(body or {}).encode("utf-8") + req = urllib.request.Request( + url, data=data, method=method, + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8") + return (True, (_json.loads(raw) if raw.strip() else {}), "") + except urllib.error.HTTPError as e: + detail = f"HTTP {e.code}" + if e.code in (401, 403): + detail = f"AUTH: HTTP {e.code} (run `az login` / check access)" + else: + try: + msg = _json.loads(e.read().decode("utf-8")).get("message") + if msg: + detail = f"HTTP {e.code}: {str(msg)[:200]}" + except Exception: + pass + return (False, None, detail) + except (urllib.error.URLError, ValueError, TimeoutError) as e: + return (False, None, f"REST {method} failed: {e}") + + def _tag_value(tags, key): """Return the value of a `key=value` build tag (e.g. RC-ECS=1678863 → '1678863'), or None. Case-sensitive key match; first match wins.""" diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py new file mode 100644 index 00000000..62e27bf6 --- /dev/null +++ b/release-agent/tools/testplans.py @@ -0,0 +1,180 @@ +"""ADO Test-Plan operations for Phase 3 (bug_bash) — the two `clone_plans_*` steps. + +Two DIFFERENT release procedures, per the team docs: + + * BROKER — COPY the master test plan (a real ADO "Copy Test Plan" / CloneOperation). + Master: plan 2007357 / rootSuite 2007358 ("Android Monthly Release Master Test + Plan", area 'Engineering\\Auth Client\\Broker\\Android'). Each release clones it to + a new plan "Android Monthly Release - ", REFERENCING the existing test + cases (the ADO clone default — it shares test cases, doesn't duplicate them). + Doc: eng.ms/.../internal-release-checklist/test-plans + + * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing + "MSAuthenticator Test Passes" plan (714514 / rootSuite 714515), named after the + release "Android/release/MM/YYYY", whose WIQL selects the Android bug-bash test + cases (tag 'Android', not Closed, not 'IgnoreOnPrem' — mirrors the live suite + 3016608 "Android/release/08/2024"). We STOP after creating the suite — assigning + testers is a later, manual step. + Doc: IdentityWiki page 33580 (How to make test suite for bug bash). + +Everything shells out to `az` (bearer token) via tools.pipelines helpers and returns an +(ok, value, detail) triple. Reads are cheap; the two creates are the only writes. +""" +from __future__ import annotations + +from tools import pipelines as P + +ORG = P.ENGINEERING_ORG # https://identitydivision.visualstudio.com +PROJECT = P.ENGINEERING_PROJECT # Engineering +_API = "api-version=7.1" + +# ---- Broker: master template to clone ---- +BROKER_MASTER_PLAN = 2007357 +BROKER_MASTER_ROOT_SUITE = 2007358 +BROKER_AREA_PATH = "Engineering\\Auth Client\\Broker\\Android" +BROKER_ITERATION = "Engineering" + +# ---- Authenticator: standing plan the query-suite hangs under ---- +AUTH_PLAN = 714514 +AUTH_ROOT_SUITE = 714515 +AUTH_AREA_PATH = "Engineering\\ISP\\Identity Apps" + +_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") + + +def _split_release(release_id: str): + """'2026-08' -> (2026, 8). Raises ValueError on a malformed id.""" + y, m = str(release_id).split("-")[:2] + return int(y), int(m) + + +def broker_plan_name(release_id: str) -> str: + """The Broker clone's destination plan name, e.g. 'Android Monthly Release - Aug 2026'.""" + y, m = _split_release(release_id) + return f"Android Monthly Release - {_MONTHS[m - 1]} {y}" + + +def auth_suite_name(release_id: str) -> str: + """The Authenticator query-suite name, e.g. 'Android/release/08/2026' — mirrors the + live convention (newest suite 'Android/release/08/2024').""" + y, m = _split_release(release_id) + return f"Android/release/{m:02d}/{y}" + + +def auth_bugbash_query() -> str: + """The WIQL for the Authenticator bug-bash query-suite — the Android test cases, + excluding Closed and on-prem-only. Verbatim shape of the live suite 3016608 + ('Android/release/08/2024').""" + return ( + "select [System.Id], [System.WorkItemType], [System.Title], " + "[Microsoft.VSTS.Common.Priority], [System.AssignedTo], [System.AreaPath] " + "from WorkItems where [System.TeamProject] = @project and " + "[System.WorkItemType] in group 'Microsoft.TestCaseCategory' and " + f"[System.AreaPath] under '{AUTH_AREA_PATH}' and " + "[System.Tags] contains 'Android' and [System.State] <> 'Closed' and " + "not [System.Tags] contains 'IgnoreOnPrem'") + + +def _plan_url(plan_id, extra=""): + return f"{ORG}/{PROJECT}/_apis/testplan/plans/{plan_id}?{_API}{extra}" + + +# ---------------------------------------------------------------- reads + +def get_plan(plan_id, timeout=60): + """(ok, {id,name,areaPath,iteration,rootSuiteId}, detail) for a test plan, or block + detail. Used to confirm an already-recorded clone still exists (idempotency).""" + ok, j, d = P._ado_rest_get(_plan_url(plan_id), timeout) + if not ok: + return (False, None, d) + root = (j or {}).get("rootSuite") or {} + return (True, {"id": j.get("id"), "name": j.get("name"), + "areaPath": j.get("areaPath"), "iteration": j.get("iteration"), + "rootSuiteId": root.get("id")}, "") + + +def get_suite(plan_id, suite_id, timeout=60): + """(ok, {id,name,suiteType}, detail) for a suite under a plan.""" + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites/{suite_id}?{_API}" + ok, j, d = P._ado_rest_get(url, timeout) + if not ok: + return (False, None, d) + return (True, {"id": j.get("id"), "name": j.get("name"), + "suiteType": j.get("suiteType")}, "") + + +def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): + """Find a DIRECT child suite of `parent_suite_id` named `name` (case-insensitive). + Returns (ok, suite_id_or_None, detail). Pages through the plan's suites. A best-effort + duplicate guard for the Authenticator create.""" + want = (name or "").strip().lower() + url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?{_API}" + f"&continuationToken=") + token = "" + for _ in range(50): # hard page cap + ok, j, d = P._ado_rest_get(url + token, timeout) + if not ok: + return (False, None, d) + for s in (j or {}).get("value") or []: + if (s.get("name") or "").strip().lower() == want: + parent = s.get("parentSuite") or {} + if str(parent.get("id")) == str(parent_suite_id): + return (True, s.get("id"), "") + token = (j or {}).get("continuationToken") or "" + if not token: + break + return (True, None, "") + + +# ---------------------------------------------------------------- writes + +def clone_broker_plan(dest_name, timeout=120): + """COPY the Broker master plan to a new plan `dest_name`, referencing existing test + cases (ADO clone default). Returns (ok, new_plan_id, detail). + + Mirrors the doc's "Copy Test Plan → Reference existing test cases → Create". + """ + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation?api-version=7.1-preview.2" + body = { + # copy every suite + hierarchy; do NOT clone requirements. Not setting a + # test-case duplication flag => ADO REFERENCES the existing test cases. + "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": False, + "cloneRequirements": False, "copyComments": False}, + "destinationTestPlan": {"name": dest_name, "project": PROJECT, + "areaPath": BROKER_AREA_PATH, "iteration": BROKER_ITERATION}, + "sourceTestPlan": {"id": BROKER_MASTER_PLAN, "suiteIds": [BROKER_MASTER_ROOT_SUITE]}, + } + ok, j, d = P._ado_rest_send(url, "POST", body, timeout) + if not ok: + return (False, None, d) + dest = (j or {}).get("destinationTestPlan") or {} + pid = dest.get("id") + if not pid: + return (False, None, f"clone returned no destination plan id (state={j.get('state')})") + return (True, pid, "") + + +def create_auth_query_suite(name, query, timeout=90): + """CREATE a query-based (dynamic) test suite `name` under the Authenticator plan's + root suite, selecting the given WIQL. Returns (ok, new_suite_id, detail).""" + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/{AUTH_PLAN}/suites?api-version=7.1-preview.1" + body = {"suiteType": "dynamicTestSuite", "name": name, + "parentSuite": {"id": AUTH_ROOT_SUITE}, "queryString": query} + ok, j, d = P._ado_rest_send(url, "POST", body, timeout) + if not ok: + return (False, None, d) + sid = (j or {}).get("id") + if not sid: + return (False, None, "suite create returned no id") + return (True, sid, "") + + +# ---------------------------------------------------------------- links + +def plan_web_url(plan_id, suite_id=None): + """A human 'define' URL for a plan (optionally a suite).""" + u = f"{ORG}/{PROJECT}/_testPlans/define?planId={plan_id}" + if suite_id: + u += f"&suiteId={suite_id}" + return u From b7d5b2ad89c056c3b64c9aa9ae4077f5efe124e5 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:16:47 +0100 Subject: [PATCH 79/90] Phase 3 clone_plans: live-tested + 2 fixes from real ADO runs; name-override knob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-tested both steps against real ADO (Engineering) with TEST-prefixed names, then deleted the artifacts. The live run exposed two real bugs, now fixed: 1. Broker clone HTTP 400 "CopyAncestorHierarchy cannot be false if multiple suite ids are provided" — the clone body set copyAncestorHierarchy=false while passing the source root suite id. Set it to true; the clone now succeeds (verified: cloned plan + suites, referencing existing test cases). 2. clone_plans_auth created a DUPLICATE suite because its name-scan only saw the first page — ADO returns the suites-list continuation token in the `x-ms-continuationtoken` RESPONSE HEADER, not the body. Added pipelines._ado_rest_get_h (header-aware GET) + _ado_rest_get_all (follows the header token); find_child_suite_by_name now pages all suites and correctly finds an existing same-named suite (verified live — no duplicate). Adds a `name` mock knob to both steps to override the derived plan/suite name (enables safe 'TEST ...' live runs and staging). Extends the test net-guard to _ado_rest_get_h / _ado_rest_send. New offline tests: name override + header-continuation paging. Suite 204. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../steps/bug_bash/clone_plans_auth.py | 5 +- .../steps/bug_bash/clone_plans_broker.py | 5 +- release-agent/tests/conftest.py | 2 + release-agent/tests/test_engine.py | 35 ++++++++++++++ release-agent/tools/pipelines.py | 48 +++++++++++++++++++ release-agent/tools/testplans.py | 33 ++++++------- 6 files changed, 107 insertions(+), 21 deletions(-) diff --git a/release-agent/steps/bug_bash/clone_plans_auth.py b/release-agent/steps/bug_bash/clone_plans_auth.py index 10582d48..c9a28f67 100644 --- a/release-agent/steps/bug_bash/clone_plans_auth.py +++ b/release-agent/steps/bug_bash/clone_plans_auth.py @@ -31,6 +31,7 @@ KIND = "agent" MOCKABLE = { + "name": {"kind": "input", "desc": "Override the suite name (e.g. a 'TEST ...' name for a safe live run)."}, "suite_id": {"kind": "input", "desc": "Pretend the query-suite already exists (this id)."}, "create_id": {"kind": "input", "desc": "Id the create should return (skip the live create)."}, "existing": {"kind": "input", "desc": "Id an existing same-named suite the name-scan finds."}, @@ -48,7 +49,9 @@ def build(state): if fail is not MISSING: return Blocked(f"clone_plans_auth: {fail}") - name = T.auth_suite_name(state.release_id) + name = mock_input("name", MISSING) + if name is MISSING: + name = T.auth_suite_name(state.release_id) step = state.get_step("bug_bash", ID) # Already created? (test-injected id, or a stored id from a prior run) → done. diff --git a/release-agent/steps/bug_bash/clone_plans_broker.py b/release-agent/steps/bug_bash/clone_plans_broker.py index 5ca9b0eb..0ef79de4 100644 --- a/release-agent/steps/bug_bash/clone_plans_broker.py +++ b/release-agent/steps/bug_bash/clone_plans_broker.py @@ -26,6 +26,7 @@ KIND = "agent" MOCKABLE = { + "name": {"kind": "input", "desc": "Override the destination plan name (e.g. a 'TEST ...' name for a safe live run)."}, "plan_id": {"kind": "input", "desc": "Pretend the clone already produced this plan id (idempotency test)."}, "clone_id": {"kind": "input", "desc": "Id the clone should return (skip the live CloneOperation)."}, "fail": {"kind": "input", "desc": "Force a Blocked with this detail (simulate an API failure)."}, @@ -41,7 +42,9 @@ def build(state): if fail is not MISSING: return Blocked(f"clone_plans_broker: {fail}") - dest = T.broker_plan_name(state.release_id) + dest = mock_input("name", MISSING) + if dest is MISSING: + dest = T.broker_plan_name(state.release_id) step = state.get_step("bug_bash", ID) # Already cloned? A test injects `plan_id` to assert idempotency; otherwise the diff --git a/release-agent/tests/conftest.py b/release-agent/tests/conftest.py index b741df60..ba998cf6 100644 --- a/release-agent/tests/conftest.py +++ b/release-agent/tests/conftest.py @@ -33,6 +33,8 @@ def _blocked(*_a, **_k): "or inject the step's input mocks).") monkeypatch.setattr(P, "_ado_rest_get", _blocked) + monkeypatch.setattr(P, "_ado_rest_get_h", _blocked) monkeypatch.setattr(P, "_ado_rest_get_text", _blocked) + monkeypatch.setattr(P, "_ado_rest_send", _blocked) monkeypatch.setattr(P, "_az_json", _blocked) yield diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 19460240..5f46b289 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2777,6 +2777,41 @@ def test_bug_bash_clone_steps_are_real_agents(): assert mod is not None and getattr(mod, "KIND", None) == "agent" and hasattr(mod, "run") +def test_clone_plans_name_override_knob(): + """The `name` mock knob overrides the derived plan/suite name (safe 'TEST ...' runs).""" + st, out = _bb_build("clone_plans_broker", + {"name": "TEST Android Monthly Release - Aug 2026", "clone_id": "1"}) + assert "TEST Android Monthly Release - Aug 2026" in out["note"] + st2, out2 = _bb_build("clone_plans_auth", + {"name": "TEST Android/release/08/2026", "existing": None, "create_id": "2"}) + assert "TEST Android/release/08/2026" in out2["note"] + + +def test_ado_rest_get_all_follows_header_continuation_token(): + """The pager concatenates every page, following the ADO `x-ms-continuationtoken` + RESPONSE HEADER (not a body field) — the bug the live clone_plans_auth test caught, + where only the first page was scanned so a same-named suite was missed → duplicate.""" + from tools import pipelines as P + pages = [ + (True, {"value": [{"id": 1}, {"id": 2}]}, {"x-ms-continuationtoken": "p2"}, ""), + (True, {"value": [{"id": 3}]}, {}, ""), # no token → last page + ] + calls = [] + + def fake_get_h(url, timeout): + calls.append(url) + return pages[len(calls) - 1] + + orig = P._ado_rest_get_h + P._ado_rest_get_h = fake_get_h + try: + ok, items, _ = P._ado_rest_get_all("https://x/_apis/y?api-version=7.1", 30) + finally: + P._ado_rest_get_h = orig + assert ok and [i["id"] for i in items] == [1, 2, 3] + assert "continuationToken=p2" in calls[1] # 2nd request carried the header token + + def test_digest_shows_rc_line_when_build_verify_active(): """When a phase that opts in (show_pipeline_runs) is active and run ids are on state, the daily digest carries a one-line RC summary; phases that don't opt in omit it.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 75fbd3a7..f010f608 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -93,6 +93,54 @@ def _ado_rest_get(url, timeout): return (False, None, f"REST GET failed: {e}") +def _ado_rest_get_h(url, timeout): + """Like _ado_rest_get but also returns the response headers: + (ok, json, headers_lower, detail). ADO returns paging tokens in the + `x-ms-continuationtoken` HEADER (not the body), so header access is needed to page.""" + az = shutil.which("az") + if az is None: + return (False, None, {}, "az CLI not found") + try: + tok = subprocess.run( + [az, "account", "get-access-token", "--resource", _ADO_RESOURCE, + "--query", "accessToken", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8") + except (subprocess.TimeoutExpired, OSError) as e: + return (False, None, {}, f"failed to get token: {e}") + if tok.returncode != 0 or not (tok.stdout or "").strip(): + return (False, None, {}, "AUTH: could not get an ADO token (run `az login`)") + token = tok.stdout.strip() + import urllib.request + import urllib.error + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + hdrs = {k.lower(): v for k, v in resp.headers.items()} + return (True, _json.loads(resp.read().decode("utf-8")), hdrs, "") + except urllib.error.HTTPError as e: + detail = f"AUTH: HTTP {e.code}" if e.code in (401, 403) else f"HTTP {e.code}" + return (False, None, {}, detail) + except (urllib.error.URLError, ValueError, TimeoutError) as e: + return (False, None, {}, f"REST GET failed: {e}") + + +def _ado_rest_get_all(url, timeout, cap_pages=60): + """GET every page of a paged ADO collection, following the `x-ms-continuationtoken` + response header. `url` must already carry its api-version (no continuationToken). + Returns (ok, all_items, detail) where all_items is the concatenated `.value` lists.""" + items, token = [], None + for _ in range(cap_pages): + u = url + (f"&continuationToken={token}" if token else "") + ok, j, hdrs, detail = _ado_rest_get_h(u, timeout) + if not ok: + return (False, None, detail) + items += (j or {}).get("value") or [] + token = hdrs.get("x-ms-continuationtoken") + if not token: + break + return (True, items, "") + + def _ado_rest_get_text(url, timeout): """GET an ADO REST url returning PLAIN TEXT (e.g. a build log). (ok, text, detail).""" az = shutil.which("az") diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 62e27bf6..3996c0c4 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -106,24 +106,18 @@ def get_suite(plan_id, suite_id, timeout=60): def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): """Find a DIRECT child suite of `parent_suite_id` named `name` (case-insensitive). - Returns (ok, suite_id_or_None, detail). Pages through the plan's suites. A best-effort - duplicate guard for the Authenticator create.""" + Returns (ok, suite_id_or_None, detail). Pages through ALL of the plan's suites + (following the ADO continuation-token header). A duplicate guard for the create.""" want = (name or "").strip().lower() - url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?{_API}" - f"&continuationToken=") - token = "" - for _ in range(50): # hard page cap - ok, j, d = P._ado_rest_get(url + token, timeout) - if not ok: - return (False, None, d) - for s in (j or {}).get("value") or []: - if (s.get("name") or "").strip().lower() == want: - parent = s.get("parentSuite") or {} - if str(parent.get("id")) == str(parent_suite_id): - return (True, s.get("id"), "") - token = (j or {}).get("continuationToken") or "" - if not token: - break + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?{_API}" + ok, suites, detail = P._ado_rest_get_all(url, timeout) + if not ok: + return (False, None, detail) + for s in suites: + if (s.get("name") or "").strip().lower() == want: + parent = s.get("parentSuite") or {} + if str(parent.get("id")) == str(parent_suite_id): + return (True, s.get("id"), "") return (True, None, "") @@ -137,9 +131,10 @@ def clone_broker_plan(dest_name, timeout=120): """ url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation?api-version=7.1-preview.2" body = { - # copy every suite + hierarchy; do NOT clone requirements. Not setting a + # copy every suite + the hierarchy; do NOT clone requirements. Not setting a # test-case duplication flag => ADO REFERENCES the existing test cases. - "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": False, + # (copyAncestorHierarchy must be true when source suiteIds are given, else ADO 400.) + "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": True, "cloneRequirements": False, "copyComments": False}, "destinationTestPlan": {"name": dest_name, "project": PROJECT, "areaPath": BROKER_AREA_PATH, "iteration": BROKER_ITERATION}, From 4ce183502f8d41183445c5f7318b793c578b0d3b Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:28:53 +0100 Subject: [PATCH 80/90] Fix clone_plans_auth query to match prod (ReleaseBugBash tag, not "not IgnoreOnPrem") MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Authenticator bug-bash query was over-including: it filtered Android test cases by "not IgnoreOnPrem" (copied from the stale 2024 suite), yielding 66 cases vs prod's 45. Live diff vs the current prod suite 3728419 ("Android release/08/13/2026") showed prod is a strict subset — the 21 extras all LACK the 'ReleaseBugBash' tag (they're ComposeTesting / ComposeSettingTesting / DarkMode cases, not the curated bug-bash set). The IDWiki doc and prod both filter on [System.Tags] contains 'ReleaseBugBash'. Because ADO's tag `contains` is a substring match, that single clause also captures month-specific 'ReleaseBugBash' tags — exactly as the doc describes. Corrected auth_bugbash_query() to filter on 'ReleaseBugBash'; verified live the new query yields 45, matching prod. Updates the test assertion, module docstring, and knowledge FAQ (adds a which-cases entry). Suite 204 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 12 +++++++----- release-agent/tests/test_engine.py | 2 +- release-agent/tools/testplans.py | 17 +++++++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index c91ac51e..d45eb6d1 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -366,10 +366,10 @@ bug_bash.clone_plans_auth: Creates a NEW query-based (dynamic) test suite under the standing 'MSAuthenticator Test Passes' plan (#714514, root suite 714515), named 'Android/release/MM/YYYY' (e.g. 'Android/release/08/2026'). Its WIQL selects the Android bug-bash test cases - (tag 'Android', area 'Engineering\\ISP\\Identity Apps', not Closed, not 'IgnoreOnPrem' - — mirrors the live suite 'Android/release/08/2024'). This is the Authenticator half - of the old clone_plans step. It STOPS after creating the suite — assigning testers is - a later, manual step. + (tag 'Android' + 'ReleaseBugBash', area 'Engineering\\ISP\\Identity Apps', not Closed + — matches the current prod suite 3728419 'Android release/08/13/2026'). This is the + Authenticator half of the old clone_plans step. It STOPS after creating the suite — + assigning testers is a later, manual step. who: "Scout runs it automatically during `next` (an agent step, via the ADO test-suite create API)." how: > Idempotent — the created suite id is stashed on the step, and it also reuses an @@ -384,7 +384,9 @@ bug_bash.clone_plans_auth: - q: "Does it assign testers?" a: "No — it stops after creating the query-based suite. Assigning testers is done later (a separate, manual step)." - q: "What names the suite?" - a: "The release: 'Android/release/MM/YYYY' — mirroring the live convention (newest was 'Android/release/08/2024')." + a: "The release: 'Android/release/MM/YYYY'." + - q: "Which test cases does it include?" + a: "The curated bug-bash set — Android test cases tagged 'ReleaseBugBash' (the tag also matches month-specific 'ReleaseBugBash' tags), not Closed. This matches prod; a broad 'all Android' filter would over-include (e.g. ComposeTesting/DarkMode cases)." # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 5f46b289..ff16f933 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2728,7 +2728,7 @@ def test_testplans_names_and_query(): assert T.broker_plan_name("2026-08") == "Android Monthly Release - Aug 2026" assert T.auth_suite_name("2026-08") == "Android/release/08/2026" q = T.auth_bugbash_query() - assert "contains 'Android'" in q and "IgnoreOnPrem" in q and "Identity Apps" in q + assert "contains 'Android'" in q and "contains 'ReleaseBugBash'" in q and "Identity Apps" in q def test_clone_plans_broker_clones_then_idempotent(): diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 3996c0c4..66660be1 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -12,8 +12,8 @@ * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" plan (714514 / rootSuite 714515), named after the release "Android/release/MM/YYYY", whose WIQL selects the Android bug-bash test - cases (tag 'Android', not Closed, not 'IgnoreOnPrem' — mirrors the live suite - 3016608 "Android/release/08/2024"). We STOP after creating the suite — assigning + cases (tag 'Android' + 'ReleaseBugBash', not Closed — matches the current prod suite + 3728419 "Android release/08/13/2026"). We STOP after creating the suite — assigning testers is a later, manual step. Doc: IdentityWiki page 33580 (How to make test suite for bug bash). @@ -63,9 +63,14 @@ def auth_suite_name(release_id: str) -> str: def auth_bugbash_query() -> str: - """The WIQL for the Authenticator bug-bash query-suite — the Android test cases, - excluding Closed and on-prem-only. Verbatim shape of the live suite 3016608 - ('Android/release/08/2024').""" + """The WIQL for the Authenticator bug-bash query-suite — the Android test cases + curated for the release bug bash (tag 'ReleaseBugBash'), excluding Closed. Matches + the current prod suite (e.g. 'Android release/08/13/2026', suite 3728419). + + Per the IDWiki doc, the always-included cases carry tag 'ReleaseBugBash'; month- + specific cases carry 'ReleaseBugBash' (e.g. 'ReleaseBugBashAug'). Because ADO's + `[System.Tags] contains 'X'` is a substring match, the single 'ReleaseBugBash' clause + captures BOTH — no separate month clause needed.""" return ( "select [System.Id], [System.WorkItemType], [System.Title], " "[Microsoft.VSTS.Common.Priority], [System.AssignedTo], [System.AreaPath] " @@ -73,7 +78,7 @@ def auth_bugbash_query() -> str: "[System.WorkItemType] in group 'Microsoft.TestCaseCategory' and " f"[System.AreaPath] under '{AUTH_AREA_PATH}' and " "[System.Tags] contains 'Android' and [System.State] <> 'Closed' and " - "not [System.Tags] contains 'IgnoreOnPrem'") + "[System.Tags] contains 'ReleaseBugBash'") def _plan_url(plan_id, extra=""): From a7cf05bcd0d5dae2f3e28c1f824081522b18999a Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:34:17 +0100 Subject: [PATCH 81/90] clone_plans_auth: name suite 'Android release/MM/DD/YYYY' from the CCD (match prod) The Authenticator suite name now follows the prod/IDWiki convention exactly - "Android release/MM/DD/YYYY" with the day taken from the release CCD (e.g. CCD 2026-08-13 -> "Android release/08/13/2026", matching prod suite 3728419). Was the day-less, extra-slash "Android/release/08/2026". auth_suite_name(ccd) now derives from the CCD date; the step passes state.ccd and blocks with a clear message if no CCD is set (the name needs the day). Updates the docstrings, knowledge FAQ, and tests; adds a no-CCD block test. Verified live (name + suite create). Suite 205 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 6 +++--- release-agent/steps/bug_bash/clone_plans_auth.py | 8 ++++++-- release-agent/tests/test_engine.py | 15 +++++++++++---- release-agent/tools/testplans.py | 14 ++++++++------ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index d45eb6d1..44bf6a7c 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -364,8 +364,8 @@ bug_bash.clone_plans_auth: summary: "Create the Authenticator bug-bash query-based test suite for this release." what: > Creates a NEW query-based (dynamic) test suite under the standing 'MSAuthenticator - Test Passes' plan (#714514, root suite 714515), named 'Android/release/MM/YYYY' - (e.g. 'Android/release/08/2026'). Its WIQL selects the Android bug-bash test cases + Test Passes' plan (#714514, root suite 714515), named 'Android release/MM/DD/YYYY' + from the CCD (e.g. CCD 2026-08-13 -> 'Android release/08/13/2026'). Its WIQL selects the Android bug-bash test cases (tag 'Android' + 'ReleaseBugBash', area 'Engineering\\ISP\\Identity Apps', not Closed — matches the current prod suite 3728419 'Android release/08/13/2026'). This is the Authenticator half of the old clone_plans step. It STOPS after creating the suite — @@ -384,7 +384,7 @@ bug_bash.clone_plans_auth: - q: "Does it assign testers?" a: "No — it stops after creating the query-based suite. Assigning testers is done later (a separate, manual step)." - q: "What names the suite?" - a: "The release: 'Android/release/MM/YYYY'." + a: "From the CCD: 'Android release/MM/DD/YYYY' (e.g. CCD 2026-08-13 → 'Android release/08/13/2026'), matching prod." - q: "Which test cases does it include?" a: "The curated bug-bash set — Android test cases tagged 'ReleaseBugBash' (the tag also matches month-specific 'ReleaseBugBash' tags), not Closed. This matches prod; a broad 'all Android' filter would over-include (e.g. ComposeTesting/DarkMode cases)." diff --git a/release-agent/steps/bug_bash/clone_plans_auth.py b/release-agent/steps/bug_bash/clone_plans_auth.py index c9a28f67..c4c514b9 100644 --- a/release-agent/steps/bug_bash/clone_plans_auth.py +++ b/release-agent/steps/bug_bash/clone_plans_auth.py @@ -3,7 +3,7 @@ Per the Authenticator "How to make a test suite for bug bash" doc, each release creates a NEW query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" -plan (714514 / rootSuite 714515), named after the release ("Android/release/MM/YYYY"), +plan (714514 / rootSuite 714515), named after the release CCD ("Android release/MM/DD/YYYY"), whose WIQL selects the Android bug-bash test cases. This is the Authenticator half of the old `clone_plans` stub. @@ -51,7 +51,11 @@ def build(state): name = mock_input("name", MISSING) if name is MISSING: - name = T.auth_suite_name(state.release_id) + if not state.ccd: + return Blocked( + "clone_plans_auth: no Code Complete Date on record — can't name the suite " + "'Android release/MM/DD/YYYY'. Set the CCD first (`set-ccd`).") + name = T.auth_suite_name(state.ccd) step = state.get_step("bug_bash", ID) # Already created? (test-injected id, or a stored id from a prior run) → done. diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index ff16f933..a9652496 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2713,12 +2713,12 @@ def test_stash_mrwp_appends_new_rc_on_id_change(): # ---- Phase 3: clone_plans_broker / clone_plans_auth ---- -def _bb_build(sid, mocks, release="2026-08"): +def _bb_build(sid, mocks, release="2026-08", ccd="2026-08-13"): """Build a bug_bash step outcome offline with the given mock inputs active.""" import steps as _steps from steps.lib import mockctx from orchestrator.outcomes import as_dict - st = ReleaseState(release_id=release) + st = ReleaseState(release_id=release, ccd=ccd) with mockctx.active(mocks): return st, as_dict(_steps.get_step("bug_bash", sid).build(st)) @@ -2726,7 +2726,8 @@ def _bb_build(sid, mocks, release="2026-08"): def test_testplans_names_and_query(): from tools import testplans as T assert T.broker_plan_name("2026-08") == "Android Monthly Release - Aug 2026" - assert T.auth_suite_name("2026-08") == "Android/release/08/2026" + # suite name comes from the CCD date: 'Android release/MM/DD/YYYY' (matches prod) + assert T.auth_suite_name("2026-08-13") == "Android release/08/13/2026" q = T.auth_bugbash_query() assert "contains 'Android'" in q and "contains 'ReleaseBugBash'" in q and "Identity Apps" in q @@ -2754,7 +2755,7 @@ def test_clone_plans_auth_creates_query_suite_then_idempotent(): stored reports done without re-creating.""" st, out = _bb_build("clone_plans_auth", {"existing": None, "create_id": "778899"}) assert out["kind"] == "done" - assert "Created the Authenticator bug-bash query-suite 'Android/release/08/2026'" in out["note"] + assert "Created the Authenticator bug-bash query-suite 'Android release/08/13/2026'" in out["note"] assert "assign testers" in out["note"].lower() # stops before assigning testers assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "778899" # idempotent via injected existing suite id @@ -2769,6 +2770,12 @@ def test_clone_plans_auth_reuses_existing_same_named_suite(): assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "424242" +def test_clone_plans_auth_blocks_without_ccd(): + """The suite name needs the CCD day ('Android release/MM/DD/YYYY') — no CCD → block.""" + st, out = _bb_build("clone_plans_auth", {"existing": None, "create_id": "1"}, ccd=None) + assert out["kind"] == "blocked" and "Code Complete Date" in out["reason"] + + def test_bug_bash_clone_steps_are_real_agents(): """Both clone steps resolve to real agent modules (KIND=agent) — no longer stubs.""" import steps as _steps diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 66660be1..5eb39d5b 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -11,7 +11,7 @@ * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" plan (714514 / rootSuite 714515), named after the - release "Android/release/MM/YYYY", whose WIQL selects the Android bug-bash test + release "Android release/MM/DD/YYYY", whose WIQL selects the Android bug-bash test cases (tag 'Android' + 'ReleaseBugBash', not Closed — matches the current prod suite 3728419 "Android release/08/13/2026"). We STOP after creating the suite — assigning testers is a later, manual step. @@ -55,11 +55,13 @@ def broker_plan_name(release_id: str) -> str: return f"Android Monthly Release - {_MONTHS[m - 1]} {y}" -def auth_suite_name(release_id: str) -> str: - """The Authenticator query-suite name, e.g. 'Android/release/08/2026' — mirrors the - live convention (newest suite 'Android/release/08/2024').""" - y, m = _split_release(release_id) - return f"Android/release/{m:02d}/{y}" +def auth_suite_name(ccd: str) -> str: + """The Authenticator query-suite name from the release's Code Complete Date, e.g. + CCD '2026-08-13' -> 'Android release/08/13/2026' — matches the prod convention + (suite 3728419 'Android release/08/13/2026') and the IDWiki doc's + 'Android release/MM/DD/YYYY'. `ccd` is 'YYYY-MM-DD'.""" + y, m, d = str(ccd).split("-")[:3] + return f"Android release/{int(m):02d}/{int(d):02d}/{int(y)}" def auth_bugbash_query() -> str: From 93aa28c922ed6911e777b996a3424a9c04b05b36 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:52:06 +0100 Subject: [PATCH 82/90] clone_plans_broker: wait for async CloneOperation to finish (fix "seems empty") MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigating a broker clone that "seemed empty" revealed the real gap: ADO's CloneOperation is ASYNCHRONOUS. clone_broker_plan POSTed the clone and returned the destination plan id straight from the POST response, before the background suite/test-case copy finished — so a downstream read (or the user opening the plan) right after the step reported done could see an empty/partial plan. Verified live: a completed clone is a perfect copy of the master (45 suites: 16 static + 29 dynamic, 276 test cases), and cloneStatistics shows clonedTestCasesCount=0 / totalTestCasesCount=271 — i.e. it correctly REFERENCES existing test cases, not duplicates. The clone status lives at cloneOperationResponse.state. clone_broker_plan now polls the CloneOperation until state 'succeeded' (blocks on 'failed') before returning, so the plan is fully populated when the step reports done (measured ~14s for the master). Adds a _clone_op helper for the nested response shape and 2 tests (async-poll-to-succeeded, block-on-failed). Suite 207 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/tests/test_engine.py | 48 ++++++++++++++++++++++++++++++ release-agent/tools/testplans.py | 48 ++++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index a9652496..ab007c5d 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2794,6 +2794,54 @@ def test_clone_plans_name_override_knob(): assert "TEST Android/release/08/2026" in out2["note"] +def test_clone_broker_plan_waits_for_async_completion(): + """clone_broker_plan POSTs the CloneOperation then POLLS the op until 'succeeded' + before returning — so the plan is fully populated when the step reports done (the + async 'seems empty if read too early' gap). state lives under cloneOperationResponse.""" + from tools import pipelines as P + from tools import testplans as T + post = (True, {"destinationTestPlan": {"id": 9001}, + "cloneOperationResponse": {"opId": 77, "state": "queued"}}, "") + # op poll: inProgress, then inProgress, then succeeded + op_pages = [ + (True, {"cloneOperationResponse": {"state": "inProgress"}}, {}, ""), + (True, {"cloneOperationResponse": {"state": "inProgress"}}, {}, ""), + (True, {"cloneOperationResponse": {"state": "succeeded"}}, {}, ""), + ] + calls = {"get": 0} + + def fake_send(url, method, body, timeout): + return post + + def fake_get_h(url, timeout): + i = calls["get"]; calls["get"] += 1 + return op_pages[min(i, len(op_pages) - 1)] + + o_send, o_get = P._ado_rest_send, P._ado_rest_get_h + P._ado_rest_send, P._ado_rest_get_h = fake_send, fake_get_h + try: + ok, pid, d = T.clone_broker_plan("TEST plan", poll_secs=0) + finally: + P._ado_rest_send, P._ado_rest_get_h = o_send, o_get + assert ok and pid == 9001 and d == "" + assert calls["get"] >= 3 # polled until 'succeeded' + + +def test_clone_broker_plan_blocks_on_failed_clone_op(): + """A failed clone operation surfaces as (False, ...) so the step blocks.""" + from tools import pipelines as P + from tools import testplans as T + o_send, o_get = P._ado_rest_send, P._ado_rest_get_h + P._ado_rest_send = lambda *a, **k: (True, {"destinationTestPlan": {"id": 1}, + "cloneOperationResponse": {"opId": 5, "state": "queued"}}, "") + P._ado_rest_get_h = lambda *a, **k: (True, {"cloneOperationResponse": {"state": "failed"}}, {}, "") + try: + ok, pid, d = T.clone_broker_plan("TEST plan", poll_secs=0) + finally: + P._ado_rest_send, P._ado_rest_get_h = o_send, o_get + assert not ok and "failed" in d + + def test_ado_rest_get_all_follows_header_continuation_token(): """The pager concatenates every page, following the ADO `x-ms-continuationtoken` RESPONSE HEADER (not a body field) — the bug the live clone_plans_auth test caught, diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 5eb39d5b..8fc7c1e6 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -130,16 +130,36 @@ def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): # ---------------------------------------------------------------- writes -def clone_broker_plan(dest_name, timeout=120): +def _clone_op(op_field, *keys): + """Read a field from a CloneOperation response, which nests the live status under + `cloneOperationResponse` (state/opId/completionDate/cloneStatistics) but the ids at + the top level. Tries the nested object first, then the top level.""" + nested = (op_field or {}).get("cloneOperationResponse") or {} + for k in keys: + if nested.get(k) is not None: + return nested.get(k) + if (op_field or {}).get(k) is not None: + return op_field.get(k) + return None + + +def clone_broker_plan(dest_name, timeout=120, poll_secs=3, max_polls=40): """COPY the Broker master plan to a new plan `dest_name`, referencing existing test - cases (ADO clone default). Returns (ok, new_plan_id, detail). - - Mirrors the doc's "Copy Test Plan → Reference existing test cases → Create". + cases (ADO clone default), and WAIT for the async clone to finish. Returns + (ok, new_plan_id, detail). + + ADO's CloneOperation is asynchronous: the POST returns a destination plan id while the + suite/test-case copy runs in the background. We poll the operation until its state is + 'succeeded' before returning, so the plan is fully populated when the step reports done + (a too-early read would otherwise show an empty/partial plan). Mirrors the doc's + "Copy Test Plan → Reference existing test cases → Create". """ + import time url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation?api-version=7.1-preview.2" body = { # copy every suite + the hierarchy; do NOT clone requirements. Not setting a - # test-case duplication flag => ADO REFERENCES the existing test cases. + # test-case duplication flag => ADO REFERENCES the existing test cases + # (cloneStatistics.clonedTestCasesCount stays 0 — shared, not duplicated). # (copyAncestorHierarchy must be true when source suiteIds are given, else ADO 400.) "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": True, "cloneRequirements": False, "copyComments": False}, @@ -153,7 +173,23 @@ def clone_broker_plan(dest_name, timeout=120): dest = (j or {}).get("destinationTestPlan") or {} pid = dest.get("id") if not pid: - return (False, None, f"clone returned no destination plan id (state={j.get('state')})") + return (False, None, f"clone returned no destination plan id (state={_clone_op(j, 'state')})") + op_id = _clone_op(j, "opId") + + # Poll the operation to completion so the plan is populated before we report done. + if op_id: + op_url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation/{op_id}" + f"?api-version=7.1-preview.2") + for _ in range(max_polls): + ok_o, op, _h, d_o = P._ado_rest_get_h(op_url, 60) + if not ok_o: + break # can't read op → best-effort, return pid + state = str(_clone_op(op, "state") or "").lower() + if state in ("succeeded", "completed"): + break + if state == "failed": + return (False, None, f"clone operation failed (op {op_id})") + time.sleep(poll_secs) return (True, pid, "") From 10f8f9b13823c766b95e4e2283a203e42fc18d4e Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Sat, 22 Aug 2026 01:40:46 +0100 Subject: [PATCH 83/90] =?UTF-8?q?Phase=203:=20distribute=5Ftests=20step=20?= =?UTF-8?q?=E2=80=94=20even=20bug-bash=20test=20distribution=20across=20th?= =?UTF-8?q?e=20team?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the test-distribution step (the "next step" after the two clone steps). Combines the two manual test sets and splits them EVENLY across the eligible team, preserving existing assignments where possible. - tools/distribution.py: pure distribute() (even +/-1, preference-preserving: the +1 slots go to the people with the most default tests) + eligible_testers() + live gatherers (Graph roster, Broker "Manual Tests (Android Broker)" subtree, Auth ReleaseBugBash minus Automated/Blocked, System.AssignedTo read, set_assigned_to write). - config/distribution.yaml: roster DL, always-excluded (jialh/moghosh/veenasoman), test-set defs. On-call team id read from readiness.yaml (oncall_now.team_id = 78848) so the entry gate and this step share ONE source. - steps/bug_bash/distribute_tests.py: agent step, READ-ONLY preview — computes + stashes the plan, reports a summary; never writes. Inserted after clone_plans_auth. - distribute-tests CLI: --preview (default, read-only) / --apply (writes AssignedTo on the shared cases -> carries the preference into next release's clone) / --json. Eligible = roster - always_excluded - release owner - OCE (resolved via ICM 78848). Default + written field are both System.AssignedTo (consistent for both plans); combined into one fair split. Tested LIVE end-to-end: clone broker + create auth + preview (74 tests = 48 broker + 26 auth across 12 eligible, 6-7 each, 46 kept) + apply 74/74 + verify, then RESTORED all 74 original assignees (0 mismatches) and deleted the clones. Adds algorithm/eligible/single- source/step tests + net-guard for the Graph/WIQL/write primitives; step-info knowledge. Suite 212 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/distribution.yaml | 31 ++ release-agent/config/knowledge.yaml | 33 ++ release-agent/config/phases.yaml | 1 + .../orchestrator/commands/__init__.py | 4 +- .../orchestrator/commands/distribute.py | 118 ++++++ .../steps/bug_bash/distribute_tests.py | 132 +++++++ release-agent/tests/conftest.py | 7 + release-agent/tests/test_engine.py | 87 +++- release-agent/tools/distribution.py | 372 ++++++++++++++++++ 9 files changed, 781 insertions(+), 4 deletions(-) create mode 100644 release-agent/config/distribution.yaml create mode 100644 release-agent/orchestrator/commands/distribute.py create mode 100644 release-agent/steps/bug_bash/distribute_tests.py create mode 100644 release-agent/tools/distribution.py diff --git a/release-agent/config/distribution.yaml b/release-agent/config/distribution.yaml new file mode 100644 index 00000000..371169a0 --- /dev/null +++ b/release-agent/config/distribution.yaml @@ -0,0 +1,31 @@ +# Test-distribution config for Phase 3 (bug_bash) — the `distribute_tests` step. +# This is DATA (not code): who receives tests, who's always excluded, and how the two +# test sets are defined. The on-call team id is NOT here — it is read from +# readiness.yaml (oncall_now.team_id) so the entry gate and this step share ONE source. + +version: 1 + +# The team the tests are distributed across — a mail-enabled group (DL). Its members are +# resolved live via Microsoft Graph. +roster_group: "androididentity@microsoft.com" + +# People ALWAYS removed from distribution (in addition to the release owner + the current +# on-call engineer, which are resolved per-release). +always_excluded: + - "jialh@microsoft.com" # Jia Le He + - "moghosh@microsoft.com" # Moumita Ghosh + - "veenasoman@microsoft.com" # Veena Soman + +# The two test sets to distribute (combined into one even split across the eligible team). +broker: + # The Broker master plan's static parent suite whose subtree holds the manual tests. + # The release's CLONED plan references the same cases; we distribute those. + master_plan: 2007357 + suite_name: "Manual Tests (Android Broker)" + +authenticator: + # The Authenticator bug-bash query set (tools.testplans.auth_bugbash_query), minus test + # cases carrying any of these tags (Automated run in CI; Blocked can't be run manually). + exclude_tags: + - "Automated" + - "Blocked" diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 44bf6a7c..1ed70b8c 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -388,6 +388,39 @@ bug_bash.clone_plans_auth: - q: "Which test cases does it include?" a: "The curated bug-bash set — Android test cases tagged 'ReleaseBugBash' (the tag also matches month-specific 'ReleaseBugBash' tags), not Closed. This matches prod; a broad 'all Android' filter would over-include (e.g. ComposeTesting/DarkMode cases)." +bug_bash.distribute_tests: + summary: "Evenly distribute the manual bug-bash tests across the eligible team." + what: > + Combines the two manual test sets — Broker ('Manual Tests (Android Broker)' subtree, + ~48 cases) + Authenticator (the ReleaseBugBash query set minus tags Automated/Blocked, + ~26 cases) — and splits them EVENLY across the eligible team, keeping each test on its + current assignee where possible (preference), only reassigning to hit an even count. + Eligible = the roster DL 'androididentity@microsoft.com' MINUS the always-excluded + people (jialh, moghosh, veenasoman), the release owner, and the current on-call + engineer (OCE). Default assignee + the written field are both System.AssignedTo. + who: > + Scout computes the PREVIEW automatically during `next` (an agent step). Applying it is + a separate, explicit action you trigger — it writes System.AssignedTo, which mutates + the shared test-case work items (so this release's assignment becomes next release's + default preference). + how: > + Preview: `distribute-tests --release [--oce ]` prints the per-tester table and + stores the plan. Apply: `distribute-tests --release --apply` writes the + assignments. The OCE team id (ICM 78848) is the SAME source the entry gate's oncall_now + check uses (read from readiness.yaml — no second copy). + links: + - name: "Broker test-plans doc" + url: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/internal-release-checklist/test-plans" + - name: "How to make a test suite for bug bash (IDWiki 33580)" + url: "https://identitydivision.visualstudio.com/IdentityWiki/_wiki/wikis/IdentityWiki.wiki/33580/How-to-make-test-suite-for-bug-bash-" + faqs: + - q: "How is 'even' decided when counts don't divide evenly?" + a: "target = total ÷ eligible; the remainder people (those with the most default tests) get one extra, so heavy-default folks keep their preference." + - q: "Does applying it change the master plan?" + a: "Yes — System.AssignedTo lives on the shared test-case work items, so the new assignee shows in the master and every plan referencing them. That's intentional: it carries the preference into the next release's clone." + - q: "Who is excluded?" + a: "Always: jialh, moghosh, veenasoman. Per-release: the release owner and the current on-call engineer (OCE, resolved via ICM team 78848 — the same team the entry gate checks)." + # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the # step entries (summary/what/who/where/how/links/faqs). diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 96675dd9..013ff16e 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -65,6 +65,7 @@ phases: steps: - { id: clone_plans_broker, name: "Clone Broker test plan (copy master template)", owner: agent, maps_to: [T3] } - { id: clone_plans_auth, name: "Create Authenticator bug-bash test suite", owner: agent, maps_to: [T3] } + - { id: distribute_tests, name: "Distribute bug-bash tests across the team", owner: agent, maps_to: [T2] } - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", owner: agent, maps_to: [T2] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", owner: agent, maps_to: [T5] } diff --git a/release-agent/orchestrator/commands/__init__.py b/release-agent/orchestrator/commands/__init__.py index 1ce78330..e523bee8 100644 --- a/release-agent/orchestrator/commands/__init__.py +++ b/release-agent/orchestrator/commands/__init__.py @@ -6,7 +6,8 @@ one, so adding a command is a localized change (new/edited module only). """ from . import (release, readiness, pipeline, notify, infra_cmd, automation, - logs, lockdown, notice, step_action, localization, rc_report, rc_poll, sim) + logs, lockdown, notice, step_action, localization, rc_report, rc_poll, + distribute, sim) # Order controls how subcommands appear in --help. REGISTRARS = [ @@ -20,6 +21,7 @@ localization.register, rc_report.register, rc_poll.register, + distribute.register, sim.register, logs.register, automation.register, diff --git a/release-agent/orchestrator/commands/distribute.py b/release-agent/orchestrator/commands/distribute.py new file mode 100644 index 00000000..3ae7730a --- /dev/null +++ b/release-agent/orchestrator/commands/distribute.py @@ -0,0 +1,118 @@ +"""`distribute-tests` — preview or apply the Phase-3 bug-bash test distribution. + + --preview (default) : re-run the distribute_tests step to (re)compute the plan and print + the per-tester table. Read-only; stores the plan on the step. + --apply : write System.AssignedTo on every test case per the STORED plan + (from a prior preview). Mutates the shared work items. Records the + step done with an applied summary. + +The OCE is supplied with --oce (the skill resolves it via ICM team 78848); the owner comes +from state. --json emits the raw plan for the skill. +""" +from __future__ import annotations +import json as _json + +from orchestrator import cli_common as C +from orchestrator import mocks as mocks_mod +from orchestrator.outcomes import as_dict +from steps.lib import mockctx +from tools import distribution as D +import steps + + +def _name_map(plan): + return {} # names aren't stored on the plan; the table shows upns + + +def _print_table(plan): + counts = plan.get("counts") or {} + print(f"Eligible testers: {len(plan.get('eligible') or [])} | " + f"Broker {plan.get('broker_total')} + Auth {plan.get('auth_total')} = " + f"{plan.get('broker_total',0)+plan.get('auth_total',0)} tests | " + f"applied={plan.get('applied')}") + print(f" excluded owner: {plan.get('owner_excluded')} | OCE: {plan.get('oce_excluded') or '(none)'}") + for u in sorted(plan.get("eligible") or [], key=lambda e: -counts.get(e, 0)): + print(f" {counts.get(u,0):3} {u}") + + +def cmd_distribute_tests(args): + st = C.load_state(args.runs_root, args.release) + + if args.apply: + return _apply(args, st) + + # PREVIEW: run the step's build() with the OCE injected, persist the plan, print it. + spec = dict(mocks_mod.load_mocks().get("bug_bash.distribute_tests", {})) + if args.oce: + spec["oce"] = args.oce + with mockctx.active(spec): + out = as_dict(steps.get_step("bug_bash", "distribute_tests").build(st)) + C.save_state(st, args.runs_root, args.release) + if out["kind"] == "blocked": + print(_json.dumps({"error": out["reason"]}) if args.json else f"BLOCKED: {out['reason']}") + return 1 + plan = (st.get_step("bug_bash", "distribute_tests").data or {}).get("plan") or {} + if args.json: + print(_json.dumps(plan, indent=2)) + else: + print(out["note"]); print(); _print_table(plan) + return 0 + + +def _apply(args, st): + plan = (st.get_step("bug_bash", "distribute_tests").data or {}).get("plan") or {} + assignments = plan.get("assignments") or {} + if not assignments: + print("No stored distribution plan — run the preview first " + "(distribute-tests --release ).") + return 1 + if plan.get("applied"): + print("This distribution was already applied.") + return 0 + + ok_n, fail = 0, [] + for key, upn in assignments.items(): + case_id = key.split(":", 1)[1] # 'B:123' / 'A:123' -> '123' + ok, detail = D.set_assigned_to(case_id, upn) + if ok: + ok_n += 1 + else: + fail.append((case_id, upn, detail)) + if str(detail).startswith("AUTH"): + break # stop on auth failure — nothing will work + + plan["applied"] = not fail + step = st.get_step("bug_bash", "distribute_tests") + step.data = dict(step.data or {}); step.data["plan"] = plan + st.set_step("bug_bash", "distribute_tests", step) + C.save_state(st, args.runs_root, args.release) + + if fail: + C.emit(args.runs_root, args.release, + f"[distribute] applied {ok_n}/{len(assignments)}; {len(fail)} failed", kind="step") + print(f"Applied {ok_n}/{len(assignments)} assignments; {len(fail)} FAILED:") + for cid, upn, d in fail[:10]: + print(f" case {cid} -> {upn}: {d}") + return 2 + # mark the step done on a clean apply + from orchestrator.engine import Orchestrator + st2, orch = C.load_orch(args.runs_root, args.release, args.config) + orch.record_scout_step("bug_bash", "distribute_tests", "pass", + f"Assigned {ok_n} bug-bash tests across {len(plan.get('eligible') or [])} testers.") + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f"[distribute] assigned {ok_n} tests across {len(plan.get('eligible') or [])} testers", kind="step") + print(f"Applied {ok_n} assignments across {len(plan.get('eligible') or [])} testers.") + return 0 + + +def register(sub): + p = sub.add_parser("distribute-tests", + help="Preview or apply the Phase-3 bug-bash test distribution") + p.add_argument("--release", required=True) + p.add_argument("--oce", default=None, + help="On-call engineer identifier to exclude (skill resolves via ICM 78848)") + p.add_argument("--apply", action="store_true", + help="Write System.AssignedTo per the stored plan (mutates shared work items)") + p.add_argument("--json", action="store_true", help="Emit the raw plan JSON") + p.set_defaults(func=cmd_distribute_tests) diff --git a/release-agent/steps/bug_bash/distribute_tests.py b/release-agent/steps/bug_bash/distribute_tests.py new file mode 100644 index 00000000..90bfb22d --- /dev/null +++ b/release-agent/steps/bug_bash/distribute_tests.py @@ -0,0 +1,132 @@ +"""Step: `distribute_tests` — evenly distribute the manual bug-bash tests across the +eligible team (Phase 3, bug_bash). + +READ-ONLY PREVIEW. This step computes the distribution (combined Broker + Authenticator +test sets, even split preserving default assignments) and STORES the resulting plan on the +step (`data.plan`), reporting a summary. It does NOT write `System.AssignedTo` — applying +the plan (which mutates the shared test-case work items) is a separate, explicit action: +`distribute-tests --release --apply`. + +Eligible = roster DL minus the always-excluded people, the release owner, and the current +on-call engineer (OCE). The OCE's team id comes from readiness.yaml (single source with the +entry gate); the OCE identity is resolved via ICM by the skill and passed in as the `oce` +input. Owner = state.owner_email. + +Depends on the two clone steps: the Broker plan id is read from clone_plans_broker's +stashed data (falls back to the master plan). If the Broker plan hasn't been cloned yet, +the step blocks. + +Mock knobs (mocks.local.yaml / tests): + oce : the on-call engineer's identifier to exclude (skill resolves via ICM). + roster : inject the team roster [{name, upn}] (skip Graph). + broker_cases : inject the Broker cases [{id, assignee}] (skip ADO). + auth_cases : inject the Authenticator cases [{id, assignee}] (skip ADO). + fail : force a Blocked with this detail. +""" +from __future__ import annotations + +from orchestrator.outcomes import Done, Blocked +from steps.lib.agent import legacy_run +from steps.lib.mockctx import mock_input, MISSING +from tools import distribution as D + +ID = "distribute_tests" +KIND = "agent" + +MOCKABLE = { + "oce": {"kind": "input", "desc": "On-call engineer identifier to exclude (skill resolves via ICM)."}, + "roster": {"kind": "input", "desc": "Inject the team roster [{name, upn}] (skip Graph)."}, + "broker_cases": {"kind": "input", "desc": "Inject Broker cases [{id, assignee}] (skip ADO)."}, + "auth_cases": {"kind": "input", "desc": "Inject Authenticator cases [{id, assignee}] (skip ADO)."}, + "fail": {"kind": "input", "desc": "Force a Blocked with this detail."}, +} + + +def _broker_plan_id(state): + """The release's cloned Broker plan id (from clone_plans_broker), or None.""" + return (state.get_step("bug_bash", "clone_plans_broker").data or {}).get("plan_id") + + +def build(state): + fail = mock_input("fail", MISSING) + if fail is not MISSING: + return Blocked(f"distribute_tests: {fail}") + + cfg = D.load_config() + + # 1) the two test sets (combined) + bcases = mock_input("broker_cases", MISSING) + if bcases is MISSING: + plan_id = _broker_plan_id(state) + if not plan_id: + return Blocked("distribute_tests: the Broker test plan hasn't been cloned yet " + "(clone_plans_broker) — run that first so the manual tests exist.") + ok, sid, d = D.find_suite_id_by_name(plan_id, cfg["broker"]["suite_name"]) + if not ok or not sid: + return Blocked(f"distribute_tests: couldn't find the '{cfg['broker']['suite_name']}' " + f"suite in Broker plan {plan_id} ({d or 'not found'}).") + ok, bcases, d = D.broker_manual_cases(plan_id, sid) + if not ok: + hint = " — run `az login`" if str(d).startswith("AUTH") else "" + return Blocked(f"distribute_tests: couldn't read Broker manual tests ({d}){hint}.") + + acases = mock_input("auth_cases", MISSING) + if acases is MISSING: + ok, acases, d = D.auth_bugbash_cases(cfg["authenticator"]["exclude_tags"]) + if not ok: + hint = " — run `az login`" if str(d).startswith("AUTH") else "" + return Blocked(f"distribute_tests: couldn't read Authenticator bug-bash tests ({d}){hint}.") + + tests = [{"id": f"B:{c['id']}", "assignee": c.get("assignee")} for c in bcases] + \ + [{"id": f"A:{c['id']}", "assignee": c.get("assignee")} for c in acases] + + # 2) eligible testers = roster - always_excluded - owner - OCE + roster = mock_input("roster", MISSING) + if roster is MISSING: + ok, roster, d = D.resolve_roster(cfg["roster_group"]) + if not ok: + hint = " — run `az login`" if str(d).startswith("AUTH") else "" + return Blocked(f"distribute_tests: couldn't resolve the roster " + f"'{cfg['roster_group']}' ({d}){hint}.") + upns = [m.get("upn") for m in roster if m.get("upn")] + owner = state.owner_email + oce = mock_input("oce", MISSING) + oce = None if oce is MISSING else oce + eligible = D.eligible_testers(upns, cfg.get("always_excluded", []), owner=owner, oce=oce) + if not eligible: + return Blocked("distribute_tests: no eligible testers after exclusions — check the " + "roster, owner, and on-call exclusions.") + + # 3) distribute (combined, even, preference-preserving) + result = D.distribute(tests, eligible) + + # 4) store the plan for the apply step; report a preview summary + name_by_upn = {m.get("upn"): m.get("name") for m in roster if m.get("upn")} + counts = result["counts"] + step = state.get_step("bug_bash", ID) + step.data = dict(step.data or {}) + step.data["plan"] = { + "assignments": result["assignments"], # {"B:"/"A:": upn} + "counts": counts, + "eligible": eligible, + "owner_excluded": owner, + "oce_excluded": oce, + "broker_total": len(bcases), + "auth_total": len(acases), + "applied": False, + } + state.set_step("bug_bash", ID, step) + + lo, hi = (min(counts.values()), max(counts.values())) if counts else (0, 0) + top = ", ".join(f"{name_by_upn.get(u, u)} {counts[u]}" + for u in sorted(eligible, key=lambda e: -counts[e])[:3]) + oce_note = f", OCE {oce}" if oce else " (OCE not resolved — pass --oce to exclude)" + return Done( + f"Distribution PREVIEW ready: {len(tests)} tests (Broker {len(bcases)} + Auth " + f"{len(acases)}) across {len(eligible)} testers — {lo}–{hi} each " + f"({result['kept']} kept, {result['reassigned']} reassigned). Excluded owner " + f"{owner}{oce_note}. e.g. {top}. Review, then apply with " + f"`distribute-tests --release {state.release_id} --apply`.") + + +run = legacy_run(build) diff --git a/release-agent/tests/conftest.py b/release-agent/tests/conftest.py index ba998cf6..dde8c7c7 100644 --- a/release-agent/tests/conftest.py +++ b/release-agent/tests/conftest.py @@ -37,4 +37,11 @@ def _blocked(*_a, **_k): monkeypatch.setattr(P, "_ado_rest_get_text", _blocked) monkeypatch.setattr(P, "_ado_rest_send", _blocked) monkeypatch.setattr(P, "_az_json", _blocked) + # tools.distribution has its own Graph + WIQL + write primitives — block those too. + try: + from tools import distribution as Dm + for fn in ("_graph_get", "_graph_token", "_ado_wiql", "set_assigned_to"): + monkeypatch.setattr(Dm, fn, _blocked) + except Exception: + pass yield diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index ab007c5d..2b2b2e51 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -75,6 +75,7 @@ def _recent_iso(): # Phase-3 bug_bash clone steps — real agents; keep flow tests offline. "bug_bash.clone_plans_broker": {"outcome": "done", "note": "broker plan cloned (test)"}, "bug_bash.clone_plans_auth": {"outcome": "done", "note": "auth suite created (test)"}, + "bug_bash.distribute_tests": {"outcome": "done", "note": "tests distributed (test)"}, } @@ -638,9 +639,9 @@ def test_holds_at_first_hold(): assert actions[-1].step == "ui_failures" # Phases 0-2 gateless (rc_report auto); first hold is Phase-3 ui_failures # auto steps that RUN before the first hold: Phase-0 breaking/cg/cron/wiki (4) + # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout - # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/coordinate - # stubs (3). The Phase-1 scout steps are pre-recorded by _orch's _clear_ccd_scout (not "ran"). - assert sum(1 for a in actions if a.kind == "ran") == 12 + # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/ + # distribute_tests/coordinate stubs (4). The Phase-1 scout steps are pre-recorded. + assert sum(1 for a in actions if a.kind == "ran") == 13 def test_gate_blocks_until_approved(): @@ -2784,6 +2785,86 @@ def test_bug_bash_clone_steps_are_real_agents(): assert mod is not None and getattr(mod, "KIND", None) == "agent" and hasattr(mod, "run") +# ---- Phase 3: distribute_tests ---- + +def test_distribution_even_and_preserves_preference(): + """distribute() lands everyone within ±1 of the target, keeps default assignees where + possible, and gives the +1 slots to the people with the most eligible defaults.""" + from tools import distribution as D + elig = ["alice", "bob", "carmine", "dave"] + # 14 tests: alice-heavy defaults + some on an INELIGIBLE 'owner' (pooled) + tests = ([{"id": f"a{i}", "assignee": "alice"} for i in range(8)] + + [{"id": f"b{i}", "assignee": "bob"} for i in range(2)] + + [{"id": f"o{i}", "assignee": "owner"} for i in range(4)]) # owner not eligible + r = D.distribute(tests, elig) + assert sum(r["counts"].values()) == 14 + assert max(r["counts"].values()) - min(r["counts"].values()) <= 1 # even (±1) + assert r["counts"]["alice"] == 4 # 14/4 -> 3 or 4 + # alice keeps 4 of her 8 defaults; bob keeps his 2 + assert r["assignments"]["b0"] == "bob" and r["assignments"]["b1"] == "bob" + assert r["kept"] >= 6 + # every assignment is an eligible tester (owner's 4 got reassigned) + assert set(r["assignments"].values()) <= set(elig) + + +def test_eligible_testers_removes_owner_oce_and_excluded(): + from tools import distribution as D + roster = ["a@x", "jialh@microsoft.com", "b@x", "owner@x", "oce@x", "c@x"] + elig = D.eligible_testers(roster, ["jialh@microsoft.com"], owner="OWNER@x", oce="oce@X") + assert elig == ["a@x", "b@x", "c@x"] # case-insensitive owner/oce/excluded removal + + +def test_oncall_team_is_single_source_from_readiness(): + """The distribution step reads the on-call team from readiness.yaml's oncall_now item — + the SAME source the entry gate uses (no second copy to drift).""" + from tools import distribution as D + tid, tname = D.oncall_team() + assert tid == 78848 and "Android Shield" in (tname or "") + + +def _dist_build(mocks, owner="owner@microsoft.com", broker_plan="900"): + import steps as _steps + from steps.lib import mockctx + from orchestrator.outcomes import as_dict + from orchestrator.state import StepState + st = ReleaseState(release_id="2026-08", owner_email=owner) + st.set_step("bug_bash", "clone_plans_broker", StepState(status="done", data={"plan_id": broker_plan})) + with mockctx.active(mocks): + return st, as_dict(_steps.get_step("bug_bash", "distribute_tests").build(st)) + + +def test_distribute_tests_step_previews_and_stores_plan(): + """The step computes the combined distribution offline and STASHES the plan on the step + (read-only preview) — it does NOT write assignments.""" + mocks = { + "roster": [{"name": "A", "upn": "a@microsoft.com"}, {"name": "B", "upn": "b@microsoft.com"}, + {"name": "Owner", "upn": "owner@microsoft.com"}, {"name": "Oce", "upn": "oce@microsoft.com"}], + "oce": "oce@microsoft.com", + "broker_cases": [{"id": "1", "assignee": "a@microsoft.com"}, {"id": "2", "assignee": None}, + {"id": "3", "assignee": "owner@microsoft.com"}, {"id": "4", "assignee": None}], + "auth_cases": [{"id": "5", "assignee": "b@microsoft.com"}, {"id": "6", "assignee": None}], + } + st, out = _dist_build(mocks) + assert out["kind"] == "done" and "PREVIEW" in out["note"] + plan = st.get_step("bug_bash", "distribute_tests").data["plan"] + assert plan["applied"] is False + assert plan["owner_excluded"] == "owner@microsoft.com" and plan["oce_excluded"] == "oce@microsoft.com" + # 6 tests / 2 eligible (a,b; owner+oce excluded) -> 3 each + assert sorted(plan["counts"].values()) == [3, 3] + assert set(plan["assignments"].values()) == {"a@microsoft.com", "b@microsoft.com"} + + +def test_distribute_tests_blocks_without_broker_clone(): + """No cloned Broker plan → the step blocks (can't locate the manual tests).""" + import steps as _steps + from steps.lib import mockctx + from orchestrator.outcomes import as_dict + st = ReleaseState(release_id="2026-08", owner_email="o@x") + with mockctx.active({"auth_cases": [], "roster": [{"name": "A", "upn": "a@x"}]}): + out = as_dict(_steps.get_step("bug_bash", "distribute_tests").build(st)) + assert out["kind"] == "blocked" and "hasn't been cloned" in out["reason"] + + def test_clone_plans_name_override_knob(): """The `name` mock knob overrides the derived plan/suite name (safe 'TEST ...' runs).""" st, out = _bb_build("clone_plans_broker", diff --git a/release-agent/tools/distribution.py b/release-agent/tools/distribution.py new file mode 100644 index 00000000..b58824a7 --- /dev/null +++ b/release-agent/tools/distribution.py @@ -0,0 +1,372 @@ +"""Test distribution for Phase 3 (bug_bash) — the `distribute_tests` step. + +Distributes the release's manual bug-bash tests EVENLY across the eligible team while +preserving existing assignments as much as possible ("keep your preference"). Two test +sets are combined into one fair split: + + * BROKER — the test cases in the "Manual Tests (Android Broker)" subtree of the Broker + plan (the release's cloned plan references the master's cases). + * AUTHENTICATOR — the ReleaseBugBash query set (tools.testplans.auth_bugbash_query) + minus cases tagged Automated (run in CI) or Blocked (can't be run manually). + +Eligible testers = members of the roster DL (config/distribution.yaml `roster_group`) +MINUS: the always-excluded people, the release owner, and the current on-call engineer +(OCE). The OCE's team id is read from readiness.yaml (oncall_now.team_id) so the entry +gate and this step share ONE source of truth. + +Default assignment source is each test case's `System.AssignedTo` (decision: consistent +for both plans). Applying the distribution writes `System.AssignedTo` back on the case +work items — so this release's assignment becomes next release's default preference. + +The `distribute()` algorithm is a pure function (no I/O) so it is fully unit-testable; +the ADO/Graph gatherers and the write are separate. +""" +from __future__ import annotations + +import json as _json +import os +import shutil +import subprocess +import urllib.parse +import urllib.request +import urllib.error + +import yaml + +from tools import pipelines as P +from tools import testplans as T + +_CONFIG_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config") +ORG = T.ORG +PROJECT = T.PROJECT +_GRAPH = "https://graph.microsoft.com/v1.0" +_GRAPH_RESOURCE = "https://graph.microsoft.com" + + +# ----------------------------------------------------------------- config + +def load_config(path: str = None) -> dict: + """The parsed config/distribution.yaml.""" + p = path or os.path.join(_CONFIG_DIR, "distribution.yaml") + with open(p, "r", encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + + +def oncall_team(readiness_path: str = None): + """(team_id, team_name) for the on-call team — read from readiness.yaml's oncall_now + item so this step and the entry gate share ONE source. (None, None) if not configured.""" + p = readiness_path or os.path.join(_CONFIG_DIR, "readiness.yaml") + try: + with open(p, "r", encoding="utf-8") as fh: + doc = yaml.safe_load(fh) or {} + except OSError: + return (None, None) + for it in doc.get("items") or []: + if it.get("id") == "oncall_now": + return (it.get("team_id"), it.get("team_name")) + return (None, None) + + +# ----------------------------------------------------------------- pure algorithm + +def eligible_testers(roster, always_excluded, owner=None, oce=None): + """The people tests are distributed to: roster MINUS always_excluded, the owner, and + the OCE. Comparison is case-insensitive on the identifier (UPN/email). Order preserved. + `roster` is a list of identifiers; the excludes are identifiers too.""" + drop = {str(x).strip().lower() for x in (always_excluded or [])} + for x in (owner, oce): + if x: + drop.add(str(x).strip().lower()) + return [p for p in roster if str(p).strip().lower() not in drop] + + +def distribute(tests, eligible): + """Assign each test to an eligible tester, EVEN counts (±1), preserving each test's + default assignee where possible. + + `tests` — list of {"id": , "assignee": }. + `eligible` — list of eligible tester identifiers. + Returns {"assignments": {test_id: assignee}, "counts": {assignee: n}, + "targets": {assignee: n}, "kept": int, "reassigned": int}. + + Algorithm (converges to even while maximizing kept preferences): + 1. target counts: base = total//n, rem = total%n; the `rem` people with the most + eligible default tests get base+1 (so heavy-default people keep their +1). + 2. keep pass: a test stays with its default assignee if that assignee is eligible + AND still under their target. + 3. fill pass: every remaining test (default ineligible, or the default already at + target) goes to an eligible tester still under target (fewest-first, stable).""" + n = len(eligible) + if n == 0: + return {"assignments": {}, "counts": {}, "targets": {}, "kept": 0, "reassigned": 0} + elig_set = {str(e).strip().lower(): e for e in eligible} # lower -> canonical + + def canon(a): + return elig_set.get(str(a).strip().lower()) if a else None + + total = len(tests) + base, rem = divmod(total, n) + + # default eligible-test counts (for choosing who gets the +1) + default_elig = {e: 0 for e in eligible} + for t in tests: + c = canon(t.get("assignee")) + if c is not None: + default_elig[c] += 1 + # the rem people with the most eligible defaults get base+1 (ties: input order) + order = sorted(eligible, key=lambda e: (-default_elig[e], eligible.index(e))) + target = {e: (base + 1 if i < rem else base) for i, e in enumerate(order)} + + counts = {e: 0 for e in eligible} + assignments = {} + unplaced = [] + + # 1) keep pass — honor the default assignee when eligible and under target + for t in tests: + c = canon(t.get("assignee")) + if c is not None and counts[c] < target[c]: + assignments[t["id"]] = c + counts[c] += 1 + else: + unplaced.append(t) + + # 2) fill pass — place the rest on eligible testers under target (fewest-first, stable) + for t in unplaced: + pick = min(eligible, key=lambda e: (counts[e], eligible.index(e))) + assignments[t["id"]] = pick + counts[pick] += 1 + + kept = sum(1 for t in tests + if canon(t.get("assignee")) is not None + and assignments.get(t["id"]) == canon(t.get("assignee"))) + return {"assignments": assignments, "counts": counts, "targets": target, + "kept": kept, "reassigned": total - kept} + + +# ----------------------------------------------------------------- ADO / Graph I/O + +def _graph_token(timeout=60): + az = shutil.which("az") + if az is None: + return (None, "az CLI not found") + try: + tok = subprocess.run( + [az, "account", "get-access-token", "--resource", _GRAPH_RESOURCE, + "--query", "accessToken", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8") + except (subprocess.TimeoutExpired, OSError) as e: + return (None, f"failed to get token: {e}") + if tok.returncode != 0 or not (tok.stdout or "").strip(): + return (None, "AUTH: could not get a Graph token (run `az login`)") + return (tok.stdout.strip(), "") + + +def _graph_get(url, timeout=60): + token, detail = _graph_token(timeout) + if token is None: + return (False, None, detail) + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return (True, _json.loads(resp.read().decode("utf-8")), "") + except urllib.error.HTTPError as e: + detail = f"AUTH: HTTP {e.code}" if e.code in (401, 403) else f"HTTP {e.code}" + return (False, None, detail) + except (urllib.error.URLError, ValueError, TimeoutError) as e: + return (False, None, f"Graph GET failed: {e}") + + +def resolve_roster(group_mail, timeout=60): + """Resolve a mail-enabled group's member USERS to identifiers. + Returns (ok, [{name, upn}], detail). Follows @odata.nextLink; users only.""" + flt = urllib.parse.quote(f"mail eq '{group_mail}'") + ok, g, d = _graph_get( + f"{_GRAPH}/groups?$filter={flt}&$select=id,displayName", timeout) + if not ok: + return (False, None, d) + vals = (g or {}).get("value") or [] + if not vals: + return (False, None, f"group '{group_mail}' not found") + gid = vals[0]["id"] + members, url = [], (f"{_GRAPH}/groups/{gid}/members" + f"?$select=displayName,userPrincipalName,mail&$top=100") + for _ in range(20): + ok, j, d = _graph_get(url, timeout) + if not ok: + return (False, None, d) + for m in (j or {}).get("value") or []: + if (m.get("@odata.type") or "").endswith("user") or m.get("userPrincipalName"): + members.append({"name": m.get("displayName"), + "upn": m.get("userPrincipalName") or m.get("mail")}) + url = (j or {}).get("@odata.nextLink") + if not url: + break + return (True, members, "") + + +def _suite_subtree(plan_id, root_suite, timeout=90): + """All suite ids under root_suite (inclusive).""" + ok, suites, d = P._ado_rest_get_all( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?api-version=7.1", timeout) + if not ok: + return (False, None, d) + children = {} + for s in suites: + children.setdefault((s.get("parentSuite") or {}).get("id"), []).append(s["id"]) + out, stack = [], [root_suite] + while stack: + sid = stack.pop() + out.append(sid) + stack += children.get(sid, []) + return (True, out, "") + + +def _cases_assignedto(case_ids, timeout=90): + """{case_id(str): assignedTo_upn_or_None} for a batch of test-case work items.""" + out = {} + ids = [str(i) for i in case_ids if i] + for i in range(0, len(ids), 190): + batch = ",".join(ids[i:i + 190]) + url = (f"{ORG}/{PROJECT}/_apis/wit/workitems?ids={batch}" + f"&fields=System.Id,System.AssignedTo&api-version=7.1") + ok, j, _h, d = P._ado_rest_get_h(url, timeout) + if not ok: + return (False, None, d) + for w in (j or {}).get("value") or []: + a = (w.get("fields") or {}).get("System.AssignedTo") + out[str(w["fields"]["System.Id"])] = (a or {}).get("uniqueName") if isinstance(a, dict) else None + return (True, out, "") + + +def broker_manual_cases(plan_id, root_suite, timeout=90): + """(ok, [{id, assignee}], detail) — the Broker "Manual Tests (Android Broker)" subtree + cases with their default AssignedTo. `plan_id` is the release's cloned Broker plan + (or the master); the cases are the same work items either way.""" + ok, subtree, d = _suite_subtree(plan_id, root_suite, timeout) + if not ok: + return (False, None, d) + case_ids = set() + for sid in subtree: + okc, cases, dc = P._ado_rest_get_all( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/Suites/{sid}/TestCase?api-version=7.1", timeout) + if not okc: + return (False, None, dc) + for c in cases: + wid = ((c.get("workItem") or {}).get("id") + or (c.get("testCase") or {}).get("id")) + if wid: + case_ids.add(str(wid)) + okf, amap, df = _cases_assignedto(case_ids, timeout) + if not okf: + return (False, None, df) + return (True, [{"id": cid, "assignee": amap.get(cid)} for cid in sorted(case_ids)], "") + + +def find_suite_id_by_name(plan_id, name, timeout=90): + """The id of the suite named `name` in `plan_id` (case-insensitive), or None.""" + ok, suites, d = P._ado_rest_get_all( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?api-version=7.1", timeout) + if not ok: + return (False, None, d) + want = (name or "").strip().lower() + for s in suites: + if (s.get("name") or "").strip().lower() == want: + return (True, s.get("id"), "") + return (True, None, "") + + +def _wiql_ids(query, timeout=90): + ok, j, _h, d = _ado_wiql(query, timeout) + if not ok: + return (False, None, d) + return (True, [str(w["id"]) for w in (j or {}).get("workItems") or []], "") + + +def _ado_wiql(query, timeout=90): + """POST a WIQL query via the ADO REST wiql endpoint. (ok, json, headers, detail).""" + az = shutil.which("az") + if az is None: + return (False, None, {}, "az CLI not found") + tok = subprocess.run( + [az, "account", "get-access-token", "--resource", P._ADO_RESOURCE, + "--query", "accessToken", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8") + if tok.returncode != 0 or not (tok.stdout or "").strip(): + return (False, None, {}, "AUTH: could not get an ADO token (run `az login`)") + data = _json.dumps({"query": query}).encode("utf-8") + req = urllib.request.Request( + f"{ORG}/{PROJECT}/_apis/wit/wiql?api-version=7.1", data=data, method="POST", + headers={"Authorization": f"Bearer {tok.stdout.strip()}", "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return (True, _json.loads(resp.read().decode("utf-8")), {}, "") + except urllib.error.HTTPError as e: + return (False, None, {}, f"HTTP {e.code}") + except (urllib.error.URLError, ValueError, TimeoutError) as e: + return (False, None, {}, f"WIQL failed: {e}") + + +def _tags_of(case_ids, timeout=90): + """{case_id: [tag,...]} for the given cases.""" + out = {} + ids = [str(i) for i in case_ids if i] + for i in range(0, len(ids), 190): + batch = ",".join(ids[i:i + 190]) + url = (f"{ORG}/{PROJECT}/_apis/wit/workitems?ids={batch}" + f"&fields=System.Id,System.Tags,System.AssignedTo&api-version=7.1") + ok, j, _h, d = P._ado_rest_get_h(url, timeout) + if not ok: + return (False, None, d) + for w in (j or {}).get("value") or []: + f = w.get("fields") or {} + tags = [x.strip() for x in (f.get("System.Tags") or "").split(";") if x.strip()] + a = f.get("System.AssignedTo") + out[str(f["System.Id"])] = {"tags": tags, + "assignee": (a or {}).get("uniqueName") if isinstance(a, dict) else None} + return (True, out, "") + + +def auth_bugbash_cases(exclude_tags, timeout=90): + """(ok, [{id, assignee}], detail) — the Authenticator ReleaseBugBash cases minus any + carrying an excluded tag (case-insensitive), with their default AssignedTo.""" + ok, ids, d = _wiql_ids(T.auth_bugbash_query(), timeout) + if not ok: + return (False, None, d) + okt, meta, dt = _tags_of(ids, timeout) + if not okt: + return (False, None, dt) + drop = {t.strip().lower() for t in (exclude_tags or [])} + out = [] + for cid in ids: + m = meta.get(cid) or {} + if any(t.lower() in drop for t in m.get("tags", [])): + continue + out.append({"id": cid, "assignee": m.get("assignee")}) + return (True, out, "") + + +def set_assigned_to(case_id, upn, timeout=60): + """WRITE: set System.AssignedTo on a test-case work item. (ok, detail). This mutates + the shared work item (visible in the master + every plan referencing it).""" + url = f"{ORG}/{PROJECT}/_apis/wit/workitems/{case_id}?api-version=7.1" + body = [{"op": "add", "path": "/fields/System.AssignedTo", "value": upn}] + az = shutil.which("az") + if az is None: + return (False, "az CLI not found") + tok = subprocess.run( + [az, "account", "get-access-token", "--resource", P._ADO_RESOURCE, + "--query", "accessToken", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8") + if tok.returncode != 0 or not (tok.stdout or "").strip(): + return (False, "AUTH: could not get an ADO token (run `az login`)") + req = urllib.request.Request( + url, data=_json.dumps(body).encode("utf-8"), method="PATCH", + headers={"Authorization": f"Bearer {tok.stdout.strip()}", + "Content-Type": "application/json-patch+json"}) + try: + with urllib.request.urlopen(req, timeout=timeout): + return (True, "") + except urllib.error.HTTPError as e: + return (False, f"HTTP {e.code}") + except (urllib.error.URLError, ValueError, TimeoutError) as e: + return (False, f"PATCH failed: {e}") From c78baf6294fd2b836dc14c6f624be71fde0092fa Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Sat, 22 Aug 2026 02:20:38 +0100 Subject: [PATCH 84/90] =?UTF-8?q?Phase=203:=20send=5Finvite=20step=20?= =?UTF-8?q?=E2=80=94=20schedule=20Bug=20Bash=20+=20send=20Teams=20meeting?= =?UTF-8?q?=20invite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the bug-bash invite step (after distribute_tests). Scout step -> NeedsSkill(workiq_create_event) the skill executes. - templates/bug-bash-invite.html: the styled invite body. Table-based + solid bgcolor (Outlook/Teams ignore CSS gradients & many background rules) and width:100% (fills the pane, not squeezed left). Broker section (test plan + ECS/Local MRWP + local-flights JSON) and Auth App section (test suite + auth pipeline TBD). Edit the file to restyle. - tools/invite.py: pure schedule_bugbash(now) timing rule + live local-flights fetch (variable group 40) + link builders + {{TOKEN}} renderer (manual replace so the flags' braces are safe). - steps/bug_bash/send_invite.py: composes the meeting from state (Broker plan <- clone_plans_broker, Auth suite <- clone_plans_auth, ECS/Local <- pipeline_runs), applies the schedule rule, renders the template, and returns the create-event payload. Blocks if the plans aren't cloned. send_to knob redirects to you for a test. Schedule rule (agreed): after 3pm or weekend -> next business morning 09:00; before 3pm on a weekday -> later same day; weekends roll to Monday. ~2h. Recipients: androididentity@ + idnadevexciamdublin@. Tested LIVE end-to-end (rendered via the step with live flags fetch, sent to me only, twice after style fixes). Adds timing + payload + block + scout-kind tests; step-info knowledge. Suite 216 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 28 ++++ release-agent/config/phases.yaml | 1 + release-agent/steps/bug_bash/send_invite.py | 131 +++++++++++++++++++ release-agent/templates/bug-bash-invite.html | 61 +++++++++ release-agent/tests/test_engine.py | 76 ++++++++++- release-agent/tools/invite.py | 115 ++++++++++++++++ 6 files changed, 410 insertions(+), 2 deletions(-) create mode 100644 release-agent/steps/bug_bash/send_invite.py create mode 100644 release-agent/templates/bug-bash-invite.html create mode 100644 release-agent/tools/invite.py diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 1ed70b8c..ec914d35 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -421,6 +421,34 @@ bug_bash.distribute_tests: - q: "Who is excluded?" a: "Always: jialh, moghosh, veenasoman. Per-release: the release owner and the current on-call engineer (OCE, resolved via ICM team 78848 — the same team the entry gate checks)." +bug_bash.send_invite: + summary: "Schedule the combined Bug Bash and send the Teams meeting invite." + what: > + Composes the Bug Bash Teams meeting from the release's real artifacts and sends the + invite. Subject ' Release Bug Bash'; ~2h; body has the Broker section + (test plan + ECS/Local MRWP runs + the local-flights JSON from variable group 40) and + the Auth App section (test suite + auth pipeline). Rendered from + templates/bug-bash-invite.html (edit the file to restyle). + who: > + Scout composes it automatically during `next` (a scout step → workiq_create_event). + Recipients are the Android Identity DL + the Dublin CIAM alias; redirect for a test + with the send_to knob (keeps the send real, points it at you). + how: > + When: reached after 3pm or on a weekend → next business morning at 09:00; before 3pm on + a weekday → later the same day. Weekends roll to Monday. Links come from the clone steps + (Broker plan, Auth suite) and Phase 2 (ECS/Local MRWP runs); flags are fetched live from + variable group 40. Blocks if the plans haven't been cloned yet. + links: + - name: "Local flags — variable group 40" + url: "https://identitydivision.visualstudio.com/Engineering/_library?view=VariableGroupView&variableGroupId=40&path=release" + faqs: + - q: "When is it scheduled?" + a: "After 3pm or on a weekend → next business morning (9am); before 3pm on a weekday → later the same day. Weekends always roll to Monday." + - q: "Who gets invited?" + a: "The Azure Identity Android SDK / Android Identity team (androididentity@microsoft.com) and idnadevexciamdublin@microsoft.com. For a test, the send_to knob redirects it to you." + - q: "Where does the template live?" + a: "templates/bug-bash-invite.html — edit it to restyle; the step always renders from that file." + # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the # step entries (summary/what/who/where/how/links/faqs). diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 013ff16e..0c2b6998 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -66,6 +66,7 @@ phases: - { id: clone_plans_broker, name: "Clone Broker test plan (copy master template)", owner: agent, maps_to: [T3] } - { id: clone_plans_auth, name: "Create Authenticator bug-bash test suite", owner: agent, maps_to: [T3] } - { id: distribute_tests, name: "Distribute bug-bash tests across the team", owner: agent, maps_to: [T2] } + - { id: send_invite, name: "Schedule Bug Bash + send Teams invite", owner: agent, source: scout, maps_to: [T2] } - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", owner: agent, maps_to: [T2] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", owner: agent, maps_to: [T5] } diff --git a/release-agent/steps/bug_bash/send_invite.py b/release-agent/steps/bug_bash/send_invite.py new file mode 100644 index 00000000..a55fbfdf --- /dev/null +++ b/release-agent/steps/bug_bash/send_invite.py @@ -0,0 +1,131 @@ +"""Step: `send_invite` — schedule the combined Bug Bash and send the Teams meeting invite +(Phase 3, bug_bash). + +Composes the Bug Bash meeting from the release's real artifacts and returns a +NeedsSkill(workiq_create_event) the skill executes (the engine can't create calendar +events). The invite body is rendered from templates/bug-bash-invite.html (edit the file to +restyle — the step always renders from it). + +When (agreed rule, see tools.invite.schedule_bugbash): + * reached after 3pm or on a weekend -> next BUSINESS morning at 09:00 + * reached before 3pm on a weekday -> later the SAME day + Weekends roll forward to Monday. ~2h duration (extend if failures need investigation). + +Recipients (real): the config `recipients` — the Azure Identity Android SDK / Android +Identity DL + the Dublin CIAM alias. Redirect for testing with the `send_to` mock knob +(keeps the send real, points it at you, tags the subject). + +Links come from prior steps + Phase 2 (no re-discovery): + * Broker test plan <- clone_plans_broker.data.plan_id (blocks if not cloned) + * Auth test suite <- clone_plans_auth.data.suite_id (blocks if not created) + * ECS / Local MRWP <- state.pipeline_runs latest rc (TBD if unresolved) + * Local flags <- ADO variable group 40 'local-flights' (live; TBD on failure) + +Mock knobs (mocks.local.yaml / tests): + now : override the clock (ISO) so the schedule rule is deterministic in tests. + flags : inject the local-flights string (skip the live var-group fetch). + send_to : redirect the invite to these attendee(s) for testing (payload override). +""" +from __future__ import annotations + +from datetime import datetime + +from orchestrator import schedule +from orchestrator.outcomes import NeedsSkill, Blocked +from steps.lib.context import resolve_recipients +from steps.lib.mockctx import mock_input, MISSING +from tools import invite as I +from tools import testplans as T + +ID = "send_invite" +KIND = "scout" + +# The real invite recipients (the Azure Identity Android SDK / Android Identity team DL + +# the Dublin CIAM alias). Redirect for a test with the `send_to` knob. +RECIPIENTS = ["androididentity@microsoft.com", "idnadevexciamdublin@microsoft.com"] + +MOCKABLE = { + "now": {"kind": "input", "desc": "Override the clock (ISO 8601) for the schedule rule."}, + "flags": {"kind": "input", "desc": "Inject the local-flights string (skip the var-group fetch)."}, + "send_to": {"kind": "payload", "sets": "attendees", "as": "list", "tag_subject": True, + "desc": "Send the invite for real, but only to these attendee(s) (DL -> you)."}, +} + + +def _latest_rc(state): + rcs = (getattr(state, "pipeline_runs", None) or {}).get("rcs") or [] + return rcs[-1] if rcs else {} + + +def build(state): + if not state.ccd: + return Blocked("send_invite: no CCD set — can't title/schedule the Bug Bash.") + + # hard deps: both plans must exist (from the two clone steps) + broker_plan = (state.get_step("bug_bash", "clone_plans_broker").data or {}).get("plan_id") + if not broker_plan: + return Blocked("send_invite: the Broker test plan hasn't been cloned yet " + "(clone_plans_broker) — run that first.") + auth_suite = (state.get_step("bug_bash", "clone_plans_auth").data or {}).get("suite_id") + if not auth_suite: + return Blocked("send_invite: the Authenticator bug-bash suite hasn't been created yet " + "(clone_plans_auth) — run that first.") + + ccd = schedule.parse_date(state.ccd) + month_year = ccd.strftime("%B %Y") + + # when + now_raw = mock_input("now", MISSING) + if now_raw is not MISSING: + now = datetime.fromisoformat(str(now_raw).replace("Z", "+00:00")).replace(tzinfo=None) + else: + now = datetime.now() + start, end, when_note = I.schedule_bugbash(now) + + # links (Phase 2 pipeline runs — TBD if not resolved) + rc = _latest_rc(state) + ecs = (rc.get("ecs") or {}).get("run_id") + local = (rc.get("local") or {}).get("run_id") + + # local flags (live var group 40, mockable) + flags = mock_input("flags", MISSING) + if flags is MISSING: + ok, flags, _d = I.local_flights() + if not ok: + flags = None + flags_html = I.format_flags_html(flags) if flags else "<TBD — see variable group 40>" + + tokens = { + "MONTH_YEAR": month_year, + "WHEN": when_note, + "BROKER_PLAN_URL": I.testplan_url(broker_plan), + "ECS_URL": I.build_url(ecs) or "#", + "LOCAL_URL": I.build_url(local) or "#", + "LOCAL_FLAGS_HTML": flags_html, + "FLAGS_GROUP_URL": I.FLAGS_GROUP_URL, + "AUTH_PLAN_URL": I.testplan_url(T.AUTH_PLAN, auth_suite), + "AUTH_PIPELINE": "<TBD>", + } + body = I.render_invite(tokens) + + recipients, rnote, prefix = resolve_recipients(state, RECIPIENTS) + subject = f"{prefix}{month_year} Release Bug Bash" + + return NeedsSkill( + tool="workiq_create_event", + payload={ + "subject": subject, + "attendees": recipients, + "body": body, + "bodyContentType": "html", + "start": start.strftime("%Y-%m-%dT%H:%M:%S"), + "end": end.strftime("%Y-%m-%dT%H:%M:%S"), + "timeZone": getattr(state, "timezone", None) or "America/Los_Angeles", + "isOnlineMeeting": True, + }, + record_as=ID, + summary=f"Schedule the {month_year} Bug Bash ({when_note}) + invite {len(recipients)} " + f"recipient(s) ({rnote})", + note=f"invited {', '.join(recipients) if recipients else '(no recipients)'}", + outbound=True, + ) diff --git a/release-agent/templates/bug-bash-invite.html b/release-agent/templates/bug-bash-invite.html new file mode 100644 index 00000000..6937058f --- /dev/null +++ b/release-agent/templates/bug-bash-invite.html @@ -0,0 +1,61 @@ + + + + + + + + +
    +
    🐞 {{MONTH_YEAR}} Release Bug Bash
    +
    {{WHEN}}  ·  ~2 hours (may extend if failures need investigation)
    +
    + +

    Blocking time for running tests for the {{MONTH_YEAR}} Bug Bash. Please execute the test cases assigned under your name and let me know if you have any questions.

    + + + +
    + ✅ Mark pass/fail in the ADO test plan  ·  🐞 Report bugs & logs in this Teams thread  ·  🔎 Surface failures here so we can triage during the bash +
    + + + + + + + +
      +
    🔒 Broker Test Pass
    + + + + +
    Test PlanBroker test plan — {{MONTH_YEAR}}
    ECS FlightsMRWP · ECS run
    Local FlightsMRWP · Local run
    + +
    {{LOCAL_FLAGS_HTML}}
    +
    + + + + + + + +
      +
    📱 Auth App Test Pass
    + + + +
    Test PlanAuthenticator bug-bash suite — {{MONTH_YEAR}}
    Auth pipeline{{AUTH_PIPELINE}}
    +
    + +
    +
    Questions? Reply in this thread. Thanks for helping ship a solid release! 🚀
    + +
    diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 2b2b2e51..a2e2d46b 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -76,6 +76,7 @@ def _recent_iso(): "bug_bash.clone_plans_broker": {"outcome": "done", "note": "broker plan cloned (test)"}, "bug_bash.clone_plans_auth": {"outcome": "done", "note": "auth suite created (test)"}, "bug_bash.distribute_tests": {"outcome": "done", "note": "tests distributed (test)"}, + "bug_bash.send_invite": {"outcome": "done", "note": "bug bash invite sent (test)"}, } @@ -640,8 +641,8 @@ def test_holds_at_first_hold(): # auto steps that RUN before the first hold: Phase-0 breaking/cg/cron/wiki (4) + # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/ - # distribute_tests/coordinate stubs (4). The Phase-1 scout steps are pre-recorded. - assert sum(1 for a in actions if a.kind == "ran") == 13 + # distribute_tests (3) + send_invite (scout, mocked done) (1) + coordinate stub (1). + assert sum(1 for a in actions if a.kind == "ran") == 14 def test_gate_blocks_until_approved(): @@ -2865,6 +2866,77 @@ def test_distribute_tests_blocks_without_broker_clone(): assert out["kind"] == "blocked" and "hasn't been cloned" in out["reason"] +# ---- Phase 3: send_invite ---- + +def test_bugbash_schedule_rule(): + """schedule_bugbash: after-3pm/weekend -> next business 9am; before-3pm weekday -> same + day later; weekends roll to Monday.""" + from datetime import datetime + from tools import invite as I + # Fri 6pm -> Mon 9am (after 3pm + weekend skip) + s, e, _ = I.schedule_bugbash(datetime(2026, 8, 21, 18, 0)) + assert (s.year, s.month, s.day, s.hour) == (2026, 8, 24, 9) and e.hour == 11 + # Fri 10am -> same day noon + s2, _, _ = I.schedule_bugbash(datetime(2026, 8, 21, 10, 0)) + assert s2.day == 21 and s2.hour == 12 + # Sat -> Mon 9am + s3, _, _ = I.schedule_bugbash(datetime(2026, 8, 22, 11, 0)) + assert s3.day == 24 and s3.hour == 9 + # Tue 3:30pm -> Wed 9am + s4, _, _ = I.schedule_bugbash(datetime(2026, 8, 25, 15, 30)) + assert (s4.day, s4.hour) == (26, 9) + + +def _invite_state(): + from orchestrator.state import StepState + st = ReleaseState(release_id="2026-08", ccd="2026-08-13", + owner_email="owner@microsoft.com", timezone="America/Los_Angeles") + st.set_step("bug_bash", "clone_plans_broker", StepState(status="done", data={"plan_id": 3730001})) + st.set_step("bug_bash", "clone_plans_auth", StepState(status="done", data={"suite_id": 3730002})) + st.pipeline_runs = {"rcs": [{"rc": 1, "ecs": {"run_id": "1678863"}, "local": {"run_id": "1678864"}}]} + return st + + +def test_send_invite_composes_create_event(): + """send_invite returns a NeedsSkill(workiq_create_event) with the meeting, real + recipients, and a body carrying every release link + the injected flags.""" + import steps as _steps + from steps.lib import mockctx + from orchestrator.outcomes import as_dict + st = _invite_state() + with mockctx.active({"now": "2026-08-21T18:00:00", + "flags": "{EnableBrowserSso:true,UseKdfVersion2:true}"}): + out = as_dict(_steps.get_step("bug_bash", "send_invite").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "workiq_create_event" + p = out["payload"] + assert p["subject"] == "August 2026 Release Bug Bash" + assert p["attendees"] == ["androididentity@microsoft.com", "idnadevexciamdublin@microsoft.com"] + assert p["start"] == "2026-08-24T09:00:00" and p["end"] == "2026-08-24T11:00:00" + assert p["isOnlineMeeting"] is True and p["bodyContentType"] == "html" + b = p["body"] + for frag in ("planId=3730001", "buildId=1678863", "buildId=1678864", + "planId=714514&suiteId=3730002", "EnableBrowserSso", + "variableGroupId=40", "August 2026"): + assert frag in b, frag + assert "release-engineer-schedule" not in b # Native Auth row removed + assert out["outbound"] is True + + +def test_send_invite_blocks_without_plans(): + """No cloned plans → the invite step blocks.""" + import steps as _steps + from orchestrator.outcomes import as_dict + st = ReleaseState(release_id="2026-08", ccd="2026-08-13", owner_email="o@x") + out = as_dict(_steps.get_step("bug_bash", "send_invite").build(st)) + assert out["kind"] == "blocked" and "cloned" in out["reason"] + + +def test_send_invite_is_scout_step(): + import steps as _steps + mod = _steps.get_step("bug_bash", "send_invite") + assert mod is not None and getattr(mod, "KIND", None) == "scout" + + def test_clone_plans_name_override_knob(): """The `name` mock knob overrides the derived plan/suite name (safe 'TEST ...' runs).""" st, out = _bb_build("clone_plans_broker", diff --git a/release-agent/tools/invite.py b/release-agent/tools/invite.py new file mode 100644 index 00000000..23a6eae4 --- /dev/null +++ b/release-agent/tools/invite.py @@ -0,0 +1,115 @@ +"""Bug-bash invite helpers for Phase 3 (bug_bash) — the `send_invite` step. + +Three concerns, all here so the step stays thin: + * schedule_bugbash(now) — the deterministic "when" rule (pure, testable). + * local_flights(...) — fetch the local-flights flags JSON from ADO variable group 40. + * render_invite(...) — fill the HTML template (templates/bug-bash-invite.html) with the + release's real links + flags. + +Scheduling rule (agreed): + * Reached AFTER 3pm, or on a weekend -> next BUSINESS morning at 09:00. + * Reached before 3pm on a weekday -> later the SAME day (now + a short notice, + rounded up to the next half hour). + Weekends always roll forward to Monday. +""" +from __future__ import annotations + +import os +from datetime import datetime, timedelta + +from tools import pipelines as P + +ORG = P.ENGINEERING_ORG +PROJECT = P.ENGINEERING_PROJECT +_TPL_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "templates") + +CUTOFF_HOUR = 15 # 3pm — after this, schedule for the next business morning +MORNING_HOUR = 9 # next-day start +SAME_DAY_NOTICE_H = 2 # same-day: start this many hours out (rounded up to :30) +DURATION_HOURS = 2 +FLAGS_GROUP_ID = 40 # ADO variable group "release" holding `local-flights` + +FLAGS_GROUP_URL = (f"{ORG}/{PROJECT}/_library?view=VariableGroupView" + f"&variableGroupId={FLAGS_GROUP_ID}&path=release") + + +def _next_business_day(d): + d = d + timedelta(days=1) + while d.weekday() >= 5: # Sat=5, Sun=6 + d += timedelta(days=1) + return d + + +def _round_up_half_hour(dt): + dt = dt.replace(second=0, microsecond=0) + if dt.minute == 0 or dt.minute == 30: + return dt + if dt.minute < 30: + return dt.replace(minute=30) + return (dt.replace(minute=0) + timedelta(hours=1)) + + +def schedule_bugbash(now, duration_hours=DURATION_HOURS): + """Return (start, end, when_note) — naive local datetimes for the meeting. + + `now` is the local datetime the step runs at. See the module docstring for the rule.""" + weekend = now.weekday() >= 5 + late = now.hour >= CUTOFF_HOUR + if weekend or late: + day = _next_business_day(now.date()) + start = datetime(day.year, day.month, day.day, MORNING_HOUR, 0) + else: + start = _round_up_half_hour(now + timedelta(hours=SAME_DAY_NOTICE_H)) + end = start + timedelta(hours=duration_hours) + when = f"{start.strftime('%A, %b %d')} · {start.strftime('%-I:%M %p') if os.name != 'nt' else start.strftime('%#I:%M %p')}" \ + f"–{end.strftime('%-I:%M %p') if os.name != 'nt' else end.strftime('%#I:%M %p')}" + return (start, end, when) + + +def local_flights(group_id=FLAGS_GROUP_ID, timeout=60): + """(ok, flags_str, detail) — the raw `local-flights` value from ADO variable group + `group_id`. The value is a brace-wrapped key:value list (not strict JSON).""" + url = (f"{ORG}/{PROJECT}/_apis/distributedtask/variablegroups/{group_id}" + f"?api-version=7.1") + ok, j, _h, d = P._ado_rest_get_h(url, timeout) + if not ok: + return (False, None, d) + v = ((j or {}).get("variables") or {}).get("local-flights") + val = v.get("value") if isinstance(v, dict) else v + if not val: + return (False, None, f"variable group {group_id} has no 'local-flights' value") + return (True, val, "") + + +def format_flags_html(flags_str): + """The local-flights value → readable HTML (comma-separated, space after each comma). + Kept simple; the raw braces are preserved.""" + if not flags_str: + return "<not available>" + return (flags_str.replace("{", "{ ").replace("}", " }") + .replace(",", ", ").replace(" ", " ")) + + +def build_url(build_id): + return f"{ORG}/{PROJECT}/_build/results?buildId={build_id}" if build_id else "" + + +def testplan_url(plan_id, suite_id=None): + u = f"{ORG}/{PROJECT}/_testPlans/execute?planId={plan_id}" + if suite_id: + u += f"&suiteId={suite_id}" + return u + + +def load_template(name="bug-bash-invite.html"): + with open(os.path.join(_TPL_DIR, name), "r", encoding="utf-8") as fh: + return fh.read() + + +def render_invite(tokens, template_name="bug-bash-invite.html"): + """Fill the HTML template by replacing {{TOKEN}} markers. Manual replace (NOT + str.format) so the braces in the flags JSON are never treated as fields.""" + html = load_template(template_name) + for k, v in tokens.items(): + html = html.replace("{{" + k + "}}", str(v if v is not None else "")) + return html From 4852c6637f23b65025bf9d28fb51ec1753c341ee Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Sat, 22 Aug 2026 03:42:23 +0100 Subject: [PATCH 85/90] =?UTF-8?q?Phase=203:=20activate=5Fchat=20step=20?= =?UTF-8?q?=E2=80=94=20resolve=20&=20store=20the=20Bug=20Bash=20meeting=20?= =?UTF-8?q?chat=20id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the first of two steps for periodic bug-bash chat updates (after send_invite). A Teams meeting chat is DORMANT until activated once (first join / message / opening its Chat pane) — only then does it get a stable 19:meeting_...@thread.v2 id automation can post to. This scout step resolves + activates that chat and stores the id on the step (data.chat_id) for the upcoming periodic-update step to read. Three-tier resolution (skill-driven, mirrors the lockdown gather->record pattern): 1. SEARCH the meeting chat by topic (workiq_search_chats) — active -> done. 2. ACTIVATE via Playwright if dormant (Teams web -> Calendar -> meeting -> click Chat), then re-search for the now-active id. 3. HUMAN FALLBACK — m_ask_user asks the owner to open the meeting + click Chat / send one message, then re-resolve. record-bugbash-chat --chat-id stores it + marks the step done; without --chat-id it holds the step for the owner (attention). stored_chat_id(state) is the helper the poster step reads. Adds 3 tests (compose, block-without-ccd, store/fallback); knowledge entry. Suite 219 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 22 ++++++ release-agent/config/phases.yaml | 1 + .../orchestrator/commands/__init__.py | 3 +- .../orchestrator/commands/bugbash_chat.py | 58 +++++++++++++++ release-agent/steps/bug_bash/activate_chat.py | 70 +++++++++++++++++++ release-agent/tests/test_engine.py | 60 +++++++++++++++- 6 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 release-agent/orchestrator/commands/bugbash_chat.py create mode 100644 release-agent/steps/bug_bash/activate_chat.py diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index ec914d35..430a2f82 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -449,6 +449,28 @@ bug_bash.send_invite: - q: "Where does the template live?" a: "templates/bug-bash-invite.html — edit it to restyle; the step always renders from that file." +bug_bash.activate_chat: + summary: "Activate the Bug Bash meeting chat and store its id for the periodic updates." + what: > + A Teams meeting chat is DORMANT until it's activated once (first join, first message, + or opening its Chat pane) — only then does it get a stable 19:meeting_…@thread.v2 id + that automation can post to. This step resolves + activates that chat and stores the id + on the step (data.chat_id) so the periodic-update step can post to it. + who: > + Scout resolves it during `next` (a scout step). Three tiers: (1) SEARCH the meeting + chat by topic; (2) if dormant, ACTIVATE via Playwright (open the meeting in Teams web, + click Chat); (3) HUMAN FALLBACK — if activation fails, the owner opens the meeting and + clicks Chat / sends one message, then it's re-resolved. + how: > + The skill runs `record-bugbash-chat --release --chat-id ` to store it + and mark the step done. If it truly can't be resolved, the same command WITHOUT + --chat-id holds the step for the owner (who activates the chat, then re-runs). + faqs: + - q: "Why can't it just post to the meeting chat directly?" + a: "A freshly-scheduled meeting's chat has no thread yet — it must be activated once (join / message / open Chat). This step does that activation and captures the id so the recurring poster can reach it." + - q: "What if Playwright can't open the chat?" + a: "It falls back to the owner: open the meeting in Teams and click Chat (or send one message) to activate the thread, then re-run the step." + # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the # step entries (summary/what/who/where/how/links/faqs). diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 0c2b6998..32b922ee 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -67,6 +67,7 @@ phases: - { id: clone_plans_auth, name: "Create Authenticator bug-bash test suite", owner: agent, maps_to: [T3] } - { id: distribute_tests, name: "Distribute bug-bash tests across the team", owner: agent, maps_to: [T2] } - { id: send_invite, name: "Schedule Bug Bash + send Teams invite", owner: agent, source: scout, maps_to: [T2] } + - { id: activate_chat, name: "Activate the Bug Bash meeting chat + store its id", owner: agent, source: scout, maps_to: [T2] } - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", owner: agent, maps_to: [T2] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", owner: agent, maps_to: [T5] } diff --git a/release-agent/orchestrator/commands/__init__.py b/release-agent/orchestrator/commands/__init__.py index e523bee8..eef68a43 100644 --- a/release-agent/orchestrator/commands/__init__.py +++ b/release-agent/orchestrator/commands/__init__.py @@ -7,7 +7,7 @@ """ from . import (release, readiness, pipeline, notify, infra_cmd, automation, logs, lockdown, notice, step_action, localization, rc_report, rc_poll, - distribute, sim) + distribute, bugbash_chat, sim) # Order controls how subcommands appear in --help. REGISTRARS = [ @@ -22,6 +22,7 @@ rc_report.register, rc_poll.register, distribute.register, + bugbash_chat.register, sim.register, logs.register, automation.register, diff --git a/release-agent/orchestrator/commands/bugbash_chat.py b/release-agent/orchestrator/commands/bugbash_chat.py new file mode 100644 index 00000000..5e1a002a --- /dev/null +++ b/release-agent/orchestrator/commands/bugbash_chat.py @@ -0,0 +1,58 @@ +"""`record-bugbash-chat` — store the resolved Bug Bash meeting chat id (Phase-3 +`activate_chat` recorder seam). + +The scout step's skill resolves the meeting chat id (search → Playwright activate → human +fallback) and calls this to persist it: + * --chat-id given -> store it on the step (data.chat_id) + mark the step done. The + periodic-update step reads it back via steps.bug_bash.activate_chat.stored_chat_id. + * --chat-id omitted -> hold the step for the owner (attention): the chat couldn't be + resolved; the owner must open the meeting + click Chat to activate the thread. +""" +from __future__ import annotations + +from orchestrator import cli_common as C + +_MEETING_PREFIX = "19:meeting_" + + +def cmd_record_bugbash_chat(args): + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + chat_id = (args.chat_id or "").strip() + + if not chat_id: + detail = ("Could not resolve the Bug Bash meeting chat. Open the meeting in Teams " + "and click Chat (or send one message) to activate the thread, then re-run " + "this step.") + orch.record_scout_step("bug_bash", "activate_chat", "attention", detail) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, f"[attention] activate_chat: {detail}", kind="step") + print(detail) + return 2 + + if not chat_id.startswith(_MEETING_PREFIX): + # not fatal — warn but still store (allow non-meeting chat ids for flexibility/tests) + print(f"warning: chat id '{chat_id}' doesn't look like a meeting chat " + f"({_MEETING_PREFIX}…@thread.v2); storing anyway.") + + orch.record_scout_step("bug_bash", "activate_chat", "pass", + f"Bug Bash meeting chat ready: {chat_id}") + step = orch.state.get_step("bug_bash", "activate_chat") + step.data = dict(step.data or {}) + step.data["chat_id"] = chat_id + step.by = "scout" + orch.state.set_step("bug_bash", "activate_chat", step) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f"[ok] activate_chat: meeting chat stored ({chat_id})", kind="step") + print(f"Stored Bug Bash meeting chat id: {chat_id}") + return 0 + + +def register(sub): + p = sub.add_parser("record-bugbash-chat", + help="Store the resolved Bug Bash meeting chat id (Phase-3 activate_chat)") + p.add_argument("--release", required=True) + p.add_argument("--chat-id", default=None, dest="chat_id", + help="The meeting chat id (19:meeting_…@thread.v2). Omit to hold for the owner.") + p.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + p.set_defaults(func=cmd_record_bugbash_chat) diff --git a/release-agent/steps/bug_bash/activate_chat.py b/release-agent/steps/bug_bash/activate_chat.py new file mode 100644 index 00000000..6fd379d5 --- /dev/null +++ b/release-agent/steps/bug_bash/activate_chat.py @@ -0,0 +1,70 @@ +"""Step: `activate_chat` — resolve & activate the Bug Bash meeting chat, store its id +(Phase 3, bug_bash). + +The periodic bug-bash updates (next step) post to the meeting's Teams chat. But a Teams +meeting chat is DORMANT until it's activated once (first join, first message, or opening +its Chat pane) — only then does it get a stable `19:meeting_…@thread.v2` id that the +automated poster can reach via workiq_send_chat_message. + +This scout step resolves + activates that chat and stores the id on the step +(`data.chat_id`) so the poller step can read it. Three-tier resolution (skill-driven): + + 1. SEARCH — find the meeting chat by topic (workiq_search_chats). Already active → done. + 2. ACTIVATE (Playwright) — if dormant, open Teams web (signed in) → Calendar → the + meeting → click Chat (materializes the thread), then re-search for the id. + 3. HUMAN FALLBACK — if activation fails, ask the owner to open the meeting and click + Chat / send one message, then re-search (or accept a pasted chat id). + +The skill runs `record-bugbash-chat --release --chat-id ` to store it (or +without --chat-id to hold the step for the owner when it truly can't be resolved). +""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator.outcomes import NeedsSkill, Blocked + +ID = "activate_chat" +KIND = "scout" + + +def stored_chat_id(state): + """The resolved Bug Bash meeting chat id, or None — read by the periodic-update step.""" + return (state.get_step("bug_bash", ID).data or {}).get("chat_id") + + +def build(state): + if not state.ccd: + return Blocked("activate_chat: no CCD set — can't identify the Bug Bash meeting.") + month_year = schedule.parse_date(state.ccd).strftime("%B %Y") + topic = f"{month_year} Release Bug Bash" + + instructions = ( + f"Resolve and activate the Bug Bash meeting chat, then store its id.\n" + f"1. SEARCH: `workiq_search_chats` for a meeting chat whose topic is " + f"'{topic}' (chatType 'meeting', id like 19:meeting_…@thread.v2). If found, that's " + f"the id — go to step 4.\n" + f"2. ACTIVATE (Playwright) if not found — the chat is dormant: open Teams web " + f"(https://teams.microsoft.com/v2/, already signed in) → Calendar → next week if " + f"needed → click the '{topic}' meeting → click the **Chat** button in the peek " + f"(this 'turns on' the meeting chat). Then re-run `workiq_search_chats` for " + f"'{topic}' to get the now-active id.\n" + f"3. HUMAN FALLBACK if Playwright can't activate it: use `m_ask_user` to ask the " + f"owner to open the '{topic}' meeting in Teams and click **Chat** (or send one " + f"message) to activate the thread — then re-search, or accept a chat id they paste.\n" + f"4. Run `record-bugbash-chat --release {state.release_id} --chat-id ''`. If it " + f"truly can't be resolved, run it WITHOUT --chat-id to hold the step for the owner." + ) + + return NeedsSkill( + tool="record-bugbash-chat", # follow-up recorder command + payload={ + "release": state.release_id, + "chat_id": None, + "followup_command": (f"record-bugbash-chat --release {state.release_id} " + f"--chat-id ''"), + "_gather": {"meeting_topic": topic, "instructions": instructions}, + }, + record_as=ID, + summary=f"Resolve + activate the '{topic}' meeting chat, then store its id", + note="awaiting meeting-chat resolution (search → Playwright activate → human fallback)", + ) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index a2e2d46b..605a10a0 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -77,6 +77,7 @@ def _recent_iso(): "bug_bash.clone_plans_auth": {"outcome": "done", "note": "auth suite created (test)"}, "bug_bash.distribute_tests": {"outcome": "done", "note": "tests distributed (test)"}, "bug_bash.send_invite": {"outcome": "done", "note": "bug bash invite sent (test)"}, + "bug_bash.activate_chat": {"outcome": "done", "note": "meeting chat activated (test)"}, } @@ -641,8 +642,9 @@ def test_holds_at_first_hold(): # auto steps that RUN before the first hold: Phase-0 breaking/cg/cron/wiki (4) + # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/ - # distribute_tests (3) + send_invite (scout, mocked done) (1) + coordinate stub (1). - assert sum(1 for a in actions if a.kind == "ran") == 14 + # distribute_tests (3) + send_invite + activate_chat (scout, mocked done) (2) + + # coordinate stub (1). + assert sum(1 for a in actions if a.kind == "ran") == 15 def test_gate_blocks_until_approved(): @@ -2937,6 +2939,60 @@ def test_send_invite_is_scout_step(): assert mod is not None and getattr(mod, "KIND", None) == "scout" +# ---- Phase 3: activate_chat + record-bugbash-chat ---- + +def test_activate_chat_composes_needs_skill(): + """activate_chat is a scout step whose NeedsSkill describes the search → Playwright + activate → human fallback resolution, keyed to the meeting topic, with the recorder + follow-up command.""" + import steps as _steps + from orchestrator.outcomes import as_dict + st = ReleaseState(release_id="2026-08", ccd="2026-08-13") + out = as_dict(_steps.get_step("bug_bash", "activate_chat").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "record-bugbash-chat" + g = out["payload"]["_gather"] + assert g["meeting_topic"] == "August 2026 Release Bug Bash" + assert "Playwright" in g["instructions"] and "m_ask_user" in g["instructions"] + assert "record-bugbash-chat --release 2026-08" in out["payload"]["followup_command"] + assert _steps.get_step("bug_bash", "activate_chat").KIND == "scout" + + +def test_activate_chat_blocks_without_ccd(): + import steps as _steps + from orchestrator.outcomes import as_dict + out = as_dict(_steps.get_step("bug_bash", "activate_chat").build(ReleaseState(release_id="2026-08"))) + assert out["kind"] == "blocked" and "CCD" in out["reason"] + + +def test_record_bugbash_chat_stores_id_and_marks_done(): + """record-bugbash-chat with --chat-id stores it on the step (readable by the poller) + and marks the step done; without --chat-id it holds the step for the owner.""" + import tempfile, argparse + from orchestrator import cli_common as _C + from orchestrator.commands import bugbash_chat as BC + from steps.bug_bash.activate_chat import stored_chat_id + with tempfile.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + st = ReleaseState(release_id=rid, ccd="2026-08-13", ccd_source="confirmed") + _C.save_state(st, d, rid) + cid = "19:meeting_ABC123@thread.v2" + ns = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, chat_id=cid) + assert BC.cmd_record_bugbash_chat(ns) == 0 + again = _C.load_state(d, rid) + assert again.is_done("bug_bash", "activate_chat") + assert stored_chat_id(again) == cid + + # no chat id -> attention hold (human fallback), step not done + st2 = ReleaseState(release_id=rid, ccd="2026-08-13", ccd_source="confirmed") + _C.save_state(st2, d, rid) + ns2 = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, chat_id=None) + assert BC.cmd_record_bugbash_chat(ns2) == 2 + after = _C.load_state(d, rid) + assert not after.is_done("bug_bash", "activate_chat") + assert after.get_step("bug_bash", "activate_chat").status == "blocked" + + def test_clone_plans_name_override_knob(): """The `name` mock knob overrides the derived plan/suite name (safe 'TEST ...' runs).""" st, out = _bb_build("clone_plans_broker", From 3d1e01eae8c85911ca97a7608b6db7856b819f38 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Sat, 22 Aug 2026 04:59:54 +0100 Subject: [PATCH 86/90] Phase 3 bug bash: periodic Teams progress updates Add bugbash_updates step (first post + fire-and-continue), the 2h post-bugbash-update poller (09:00-18:00 LA, weekdays, US holidays skipped), and the bugbash_complete owner-attest gate. Updates group by AssignedTo, surface failed/blocked tests, and @mention only owners with remaining work. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/automations.yaml | 14 + release-agent/config/knowledge.yaml | 56 ++++ release-agent/config/phases.yaml | 4 +- .../orchestrator/commands/__init__.py | 3 +- .../orchestrator/commands/bugbash_update.py | 86 ++++++ release-agent/skill/reference/commands.md | 1 + .../steps/bug_bash/bugbash_updates.py | 125 +++++++++ release-agent/tests/test_engine.py | 194 +++++++++++-- release-agent/tools/bugbash.py | 264 ++++++++++++++++++ 9 files changed, 719 insertions(+), 28 deletions(-) create mode 100644 release-agent/orchestrator/commands/bugbash_update.py create mode 100644 release-agent/steps/bug_bash/bugbash_updates.py create mode 100644 release-agent/tools/bugbash.py diff --git a/release-agent/config/automations.yaml b/release-agent/config/automations.yaml index a36903d3..5f39a503 100644 --- a/release-agent/config/automations.yaml +++ b/release-agent/config/automations.yaml @@ -63,4 +63,18 @@ automations: - build_verify.rc_report # poll the in-flight RC (poll-rc): re-gate on completion, nudge at 6h purpose: "Poll a re-triggered RC every 30 min; hold while in-flight, re-apply the gate on completion, courtesy-nudge the owner at 6h" + # ON-DEMAND (not provisioned at release start). Created when `bugbash_updates` fires the + # first progress post; drives `post-bugbash-update` every 2h during working hours + # (09:00–18:00 LA, weekdays, skipping US holidays). Torn down when the tick reports + # `complete` (all tests done) or the owner attests `bugbash_complete`. Interval + # automation, so its step needs no fire_at_local. + - slug: bug-bash-update-poller + name: "Release {release} — bug-bash update poller" + phase: bug_bash + every: "2 hours" + on_demand: true + steps: + - bug_bash.bugbash_updates # post a progress update every 2h (post-bugbash-update) + purpose: "Post a Bug Bash progress update to the meeting chat every 2h (working hours); @mention owners with remaining tests; stop when all complete or the owner signs off" + diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 430a2f82..3c202eb6 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -471,6 +471,62 @@ bug_bash.activate_chat: - q: "What if Playwright can't open the chat?" a: "It falls back to the owner: open the meeting in Teams and click Chat (or send one message) to activate the thread, then re-run the step." +bug_bash.bugbash_updates: + summary: "Post the first bug-bash progress update to the meeting chat and start the 2h update poller." + what: > + Reaching this step posts an initial progress update to the Bug Bash meeting chat and + provisions the on-demand `bug-bash-update-poller` automation. Each update is grouped by + each test's owner (System.AssignedTo, set by distribute_tests, read live from BOTH the + Broker 'Manual Tests (Android Broker)' subtree and the Authenticator bug-bash suite). + A test counts as done only when it PASSED (or is N/A); not-run, failed, and blocked all + count as remaining and are LISTED (failed first, then blocked, then not-run) with links + + state so failures are surfaced, not buried. Owners with any remaining test are + @mentioned; owners who passed everything appear by name with an 'all completed' line (no + mention). It is fire-and-continue — the step completes after the first post so the phase + proceeds; the poller runs in the background. + who: > + Scout resolves it during `next` (a scout step). It gathers progress, posts the first + update via `workiq_send_chat_message` (contentType html; the payload carries `_mentions` + = [{id,upn,name}] — the skill builds the mentions array + matching tags so + owners with remaining tests are pinged), then provisions the poller. + how: > + After the first post, provision the poller (it's on-demand, so it is NOT in + `automation plan` — create it directly like the RC poller): `m_create_automation` + with name "Release — bug-bash update poller", schedule "every 2 hours", and the + prompt from this step's `automation_prompt(release, {interval: true})`; then + `automation register --id --name "..." --release --slug + bug-bash-update-poller --step bug_bash.bugbash_updates`. Each tick runs + `post-bugbash-update --release ` and acts on the decision (off_hours → nothing; + post → send content+mentions to chatId; complete → send the completion summary, then + DEREGISTER the poller). Blocks if the meeting chat isn't activated (run activate_chat). + faqs: + - q: "When do the updates run?" + a: "Every 2 hours during working hours — 09:00–18:00 America/Los_Angeles, weekdays only, skipping hardcoded US federal holidays. Outside that window a tick returns off_hours and sends nothing; it resumes at 09:00 the next working day." + - q: "When do the updates stop?" + a: "Either automatically when every test is complete (a tick returns `complete`, posts a wrap-up, and deregisters the poller) OR when the owner attests the bugbash_complete gate (the skill deregisters the poller then)." + - q: "Why group by AssignedTo instead of the test point's tester?" + a: "The ADO test-point `tester` field is NOT synced with the case's System.AssignedTo. distribute_tests sets AssignedTo, so progress is grouped by AssignedTo to reflect who actually owns each test." + +bug_bash.bugbash_complete: + summary: "The release owner attests the Bug Bash is finished — the gate that closes the bash." + what: > + A human sign-off gate: the owner confirms the assigned tests have been run (or explicitly + triaged/blocked) and the team agrees the bash is done. Approving it advances the release + out of Phase 3. It also STOPS the periodic updates. + who: > + The owner approves it (`approve` / `done --step bugbash_complete`). Until then — or until + every test auto-completes — the `bug-bash-update-poller` keeps posting every 2h. + how: > + When the owner signs off, DEREGISTER the bug-bash update poller so it stops posting: + `automation list --release --step-filter bug_bash.bugbash_updates --json` to find its + id, then `automation deregister --id `. (If all tests had already completed, the poller + deregistered itself on its last `complete` tick — deregister is idempotent/safe to skip.) + faqs: + - q: "Can I sign off before every single test passes?" + a: "Yes — this is a judgement gate. If the remaining tests are triaged/blocked and the team agrees the bash is done, approve it. It doesn't require 100% pass; that's what the sign-off is for." + - q: "What stops the 2-hour chat updates?" + a: "Two things: the poller stops itself when every test is complete, and approving this gate deregisters the poller. Either way the updates end when the bash is done." + # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the # step entries (summary/what/who/where/how/links/faqs). diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 32b922ee..144a21ef 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -68,10 +68,10 @@ phases: - { id: distribute_tests, name: "Distribute bug-bash tests across the team", owner: agent, maps_to: [T2] } - { id: send_invite, name: "Schedule Bug Bash + send Teams invite", owner: agent, source: scout, maps_to: [T2] } - { id: activate_chat, name: "Activate the Bug Bash meeting chat + store its id", owner: agent, source: scout, maps_to: [T2] } - - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", owner: agent, maps_to: [T2] } + - { id: bugbash_updates, name: "Post bug-bash progress updates (first post + 2h poller)", owner: agent, source: scout, maps_to: [T2] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", owner: agent, maps_to: [T5] } - - { id: bash_done, name: "Bug bash complete + signed off", owner: human, gate: true } + - { id: bugbash_complete, name: "Bug bash complete + signed off", owner: human, source: scout, gate: true } - id: finalize name: "Finalize & Publish" diff --git a/release-agent/orchestrator/commands/__init__.py b/release-agent/orchestrator/commands/__init__.py index eef68a43..4d9eb167 100644 --- a/release-agent/orchestrator/commands/__init__.py +++ b/release-agent/orchestrator/commands/__init__.py @@ -7,7 +7,7 @@ """ from . import (release, readiness, pipeline, notify, infra_cmd, automation, logs, lockdown, notice, step_action, localization, rc_report, rc_poll, - distribute, bugbash_chat, sim) + distribute, bugbash_chat, bugbash_update, sim) # Order controls how subcommands appear in --help. REGISTRARS = [ @@ -23,6 +23,7 @@ rc_poll.register, distribute.register, bugbash_chat.register, + bugbash_update.register, sim.register, logs.register, automation.register, diff --git a/release-agent/orchestrator/commands/bugbash_update.py b/release-agent/orchestrator/commands/bugbash_update.py new file mode 100644 index 00000000..c798aeec --- /dev/null +++ b/release-agent/orchestrator/commands/bugbash_update.py @@ -0,0 +1,86 @@ +"""`post-bugbash-update` — one tick of the bug-bash update poller (Phase-3 +`bugbash_updates`), and `record-bugbash-updates-started` — the no-op followup that keeps +the trigger step from being force-recorded after the first post. + +post-bugbash-update: gate on the working window (09:00–18:00 America/Los_Angeles, weekday, +not a US holiday), read live progress, and print a deterministic decision the poller acts +on: + off_hours — outside the working window; send nothing. + no_chat — the meeting chat isn't activated (run activate_chat). + error — couldn't read progress (detail included). + complete — every test is done: content is the completion summary; the poller posts it, + deregisters itself, and the owner signs off (bugbash_complete). + post — content (HTML) + mentions to send to chatId. + +`--now` overrides the clock for the window math (tests). `--force` skips the window gate +(the immediate first post is handled by the step itself, so this is mainly for testing). +""" +from __future__ import annotations +import json as _json +from datetime import datetime +from zoneinfo import ZoneInfo + +from orchestrator import cli_common as C +from orchestrator import schedule +from tools import bugbash as BB +from steps.bug_bash.activate_chat import stored_chat_id +from steps.bug_bash import bugbash_updates as BU + +_LA = ZoneInfo("America/Los_Angeles") + + +def cmd_post_bugbash_update(args): + now = None + if getattr(args, "now", None): + try: + now = datetime.fromisoformat(args.now.replace("Z", "+00:00")) + except ValueError: + print(_json.dumps({"decision": "error", "detail": f"bad --now: {args.now!r}"})) + return 1 + if now is None: + now = datetime.now(_LA) + now_naive = now.replace(tzinfo=None) + + if not getattr(args, "force", False) and not BB.is_working_time(now_naive): + print(_json.dumps({"decision": "off_hours", + "note": "outside 09:00–18:00 LA on a working day"})) + return 0 + + st = C.load_state(args.runs_root, args.release) + chat_id = stored_chat_id(st) + if not chat_id: + print(_json.dumps({"decision": "no_chat", + "note": "meeting chat not activated (run activate_chat)"})) + return 0 + + ok, progress, detail = BU.gather(st) + if not ok: + print(_json.dumps({"decision": "error", "detail": detail})) + return 0 + + month_year = schedule.parse_date(st.ccd).strftime("%B %Y") if st.ccd else "Bug Bash" + + if BB.all_complete(progress): + summary = (f'

    ' + f'

    🎉 {month_year} Bug Bash — all {progress["total"]} tests complete!
    ' + f'Thanks everyone. Closing out the bash; no more automated updates.

    ') + print(_json.dumps({"decision": "complete", "chatId": chat_id, "content": summary, + "total": progress["total"]})) + return 0 + + content, mentions = BB.render_update(progress, month_year, BU.plan_links(st)) + print(_json.dumps({"decision": "post", "chatId": chat_id, + "content": content, "mentions": mentions, + "done": progress["done"], "total": progress["total"], + "remaining": progress["remaining"]})) + return 0 + + +def register(sub): + p = sub.add_parser("post-bugbash-update", + help="One tick of the bug-bash update poller: gate the window, read " + "progress, print a post/complete/off_hours decision") + p.add_argument("--release", required=True) + p.add_argument("--now", default=None, help="Override 'now' (ISO 8601) for the window math") + p.add_argument("--force", action="store_true", help="Skip the working-window gate") + p.set_defaults(func=cmd_post_bugbash_update) diff --git a/release-agent/skill/reference/commands.md b/release-agent/skill/reference/commands.md index 65f3c475..297bde2b 100644 --- a/release-agent/skill/reference/commands.md +++ b/release-agent/skill/reference/commands.md @@ -42,6 +42,7 @@ _Loaded on demand. Run all from `C:\repos\android-complete\release-agent`._ | Localization: one poll | `python -m orchestrator.cli check-localization --release --complete [--logs ""]` — wait / timeout(email) / complete(post PR); acts on the printed decision | | **Phase 2 — signal a re-triggered RC** | `python -m orchestrator.cli rc-retriggered --release [--reason "..."]` — after the owner re-runs RC (flaky) or the orchestrator triggers a fresh RC (broker cherry-pick), reopens `mrwp_ecs`/`mrwp_local`/`rc_report` so Scout re-evaluates the **newest** RC. Holds `in_flight` (no action) while the run executes; the poller re-applies the gate on completion | | **Phase 2 — one RC poll** | `python -m orchestrator.cli poll-rc --release ` — advances the drain (in-flight verify re-checks live status) and prints a decision: `waiting` / `nudge` (>6h courtesy heads-up to the owner) / `resolved` (passed → deregister the poller) / `blocked` (re-gate failed) / `idle`. Driven by the on-demand `build-verify-rc-poller` automation (every 30 min) | +| **Phase 3 — one bug-bash update** | `python -m orchestrator.cli post-bugbash-update --release [--force]` — gates the working window (09:00–18:00 LA, weekdays, skips US holidays), reads live test progress, and prints a decision: `off_hours` (send nothing) / `no_chat` / `error` / `post` (send `content` HTML + `mentions` to `chatId`) / `complete` (all tests done → post the wrap-up, then **deregister the poller**). Driven by the on-demand `bug-bash-update-poller` automation (every 2 h) | | Activate conditional hotfix phase | `python -m orchestrator.cli activate --release --phase hotfix` | | **Notify** — push line if something needs me | `python -m orchestrator.cli notify [--release ] [--as-of ] [--force]` | | **Plan timed automations** | `automation plan --release [--json]` — derive the per-release CCD automations (name/schedule/steps/prompt) from `config/automations.yaml` + CCD | diff --git a/release-agent/steps/bug_bash/bugbash_updates.py b/release-agent/steps/bug_bash/bugbash_updates.py new file mode 100644 index 00000000..2a4b669b --- /dev/null +++ b/release-agent/steps/bug_bash/bugbash_updates.py @@ -0,0 +1,125 @@ +"""Step: `bugbash_updates` — post the FIRST bug-bash progress update and start the 2-hour +poller (Phase 3, bug_bash). + +Fire-and-continue (like the localization noon trigger): reaching this step posts the first +progress update to the Bug Bash meeting chat and provisions the `bugbash-update-poller` +automation; the step itself completes so the phase can proceed. The poller then posts an +update every 2 hours during working hours (09:00–18:00 America/Los_Angeles, weekdays, +skipping US holidays) via `post-bugbash-update`, and tears itself down when either every +test is complete OR the owner attests `bugbash_complete`. + +Each update is grouped by the test's owner (System.AssignedTo, set by distribute_tests): +owners with remaining tests (not-run, failed, or blocked — failed/blocked surfaced, not +hidden) are @mentioned with those tests (links + state); owners who passed everything appear +by name with an "all completed" line (no mention). A test counts as done only when it passed +(or is N/A). + +Depends on: clone_plans_broker (Broker plan id), clone_plans_auth (Auth suite id), +activate_chat (meeting chat id). Blocks if the chat hasn't been activated. + +Mock knobs (mocks.local.yaml / tests): + progress : inject the gathered progress dict (skip the live ADO reads). + send_to : redirect the post to this chat id for testing. +""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator.outcomes import NeedsSkill, Blocked, Done +from steps.lib.mockctx import mock_input, MISSING +from tools import bugbash as BB +from tools import testplans as T +from steps.bug_bash.activate_chat import stored_chat_id + +ID = "bugbash_updates" +KIND = "scout" + +BROKER_SUITE_NAME = "Manual Tests (Android Broker)" + +MOCKABLE = { + "progress": {"kind": "input", "desc": "Inject the gathered progress dict (skip ADO reads)."}, + "send_to": {"kind": "payload", "sets": "chatId", + "desc": "Post to this chat id instead of the resolved meeting chat (test)."}, +} + + +def _broker_plan(state): + return (state.get_step("bug_bash", "clone_plans_broker").data or {}).get("plan_id") + + +def _auth_suite(state): + return (state.get_step("bug_bash", "clone_plans_auth").data or {}).get("suite_id") + + +def gather(state): + """(ok, progress, detail) — live progress, or the injected `progress` mock.""" + inj = mock_input("progress", MISSING) + if inj is not MISSING: + return (True, inj, "") + bp, asuite = _broker_plan(state), _auth_suite(state) + if not bp or not asuite: + return (False, None, "the Broker plan / Auth suite aren't ready (run the clone steps).") + return BB.gather_progress(bp, BROKER_SUITE_NAME, T.AUTH_PLAN, asuite) + + +def plan_links(state): + return [ + {"name": "Broker test plan", "url": T.plan_web_url(_broker_plan(state))}, + {"name": "Authenticator suite", "url": T.plan_web_url(T.AUTH_PLAN, _auth_suite(state))}, + ] + + +def build(state): + if not state.ccd: + return Blocked("bugbash_updates: no CCD set — can't title the Bug Bash.") + chat_id = stored_chat_id(state) + if not chat_id: + return Blocked("bugbash_updates: the Bug Bash meeting chat isn't activated yet " + "(run activate_chat first).") + + ok, progress, detail = gather(state) + if not ok: + return Blocked(f"bugbash_updates: couldn't read test progress ({detail}).") + + month_year = schedule.parse_date(state.ccd).strftime("%B %Y") + if BB.all_complete(progress): + return Done(f"All {progress['total']} bug-bash tests are already complete — " + f"nothing to poll; ready for {month_year} bug bash sign-off.") + + content, mentions = BB.render_update(progress, month_year, plan_links(state)) + return NeedsSkill( + tool="workiq_send_chat_message", + payload={ + "chatId": chat_id, + "content": content, + "contentType": "html", + "_mentions": mentions, # [{id,upn,name}] — skill builds the mention array + }, + record_as=ID, + summary=(f"Post the first {month_year} bug-bash update ({progress['done']}/" + f"{progress['total']} done) to the meeting chat, then provision the 2h " + f"update poller"), + note=f"{progress['remaining']} test(s) remaining across {len(progress['owners'])} owner(s)", + outbound=True, + ) + + +def automation_prompt(release: str, spec: dict) -> str: + """Bespoke instruction for the 2-hour bug-bash update poller (interval automation).""" + if not spec.get("interval"): + return "" + return ( + f"Release {release} — bug-bash update poller (every 2h, working hours).\n" + f"Run `post-bugbash-update --release {release}` and act on the printed decision:\n" + f" • off_hours / weekend / holiday → send nothing (outside 09:00–18:00 LA on a " + f"working day).\n" + f" • post → send decision.content (HTML) to decision.chatId via " + f"`workiq_send_chat_message`. If decision.mentions is non-empty, include the " + f"@mentions (contentType html, Name tags matching the mentions " + f"array) so owners with remaining tests are pinged.\n" + f" • complete → every test is done: send decision.content (the completion summary) " + f"to decision.chatId, then DEREGISTER this poller (`automation deregister --id " + f"`) and tell the owner the bash is ready to sign off " + f"(bugbash_complete).\n" + f" • no_chat / error → surface briefly; nothing to send.\n" + f"Silently journal: `journal --release {release} --source scout --kind automation " + f"--text \"bugbash-poller: \"`.") diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 605a10a0..f883c14f 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -78,6 +78,7 @@ def _recent_iso(): "bug_bash.distribute_tests": {"outcome": "done", "note": "tests distributed (test)"}, "bug_bash.send_invite": {"outcome": "done", "note": "bug bash invite sent (test)"}, "bug_bash.activate_chat": {"outcome": "done", "note": "meeting chat activated (test)"}, + "bug_bash.bugbash_updates": {"outcome": "done", "note": "first bug-bash update posted (test)"}, } @@ -170,12 +171,12 @@ def _orch(signed=True): def _advance_to_first_gate(orch): - """Now that go_test is gone, the first real GATE is Phase-3 `bug_bash.bash_done`, + """Now that go_test is gone, the first real GATE is Phase-3 `bug_bash.bugbash_complete`, reached after the Phase-3 `ui_failures` human reminder. Drive to that reminder, clear - it, then drive to the bash_done gate.""" + it, then drive to the bugbash_complete gate.""" orch.run_until_gate() # holds at ui_failures (reminder) orch.complete_step("bug_bash", "ui_failures", "test: UI failures reviewed") - orch.run_until_gate() # holds at bash_done (gate) + orch.run_until_gate() # holds at bugbash_complete (gate) # ---- readiness entry gate ---- @@ -643,7 +644,7 @@ def test_holds_at_first_hold(): # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/ # distribute_tests (3) + send_invite + activate_chat (scout, mocked done) (2) + - # coordinate stub (1). + # bugbash_updates (scout, mocked done) (1). assert sum(1 for a in actions if a.kind == "ran") == 15 @@ -653,16 +654,16 @@ def test_gate_blocks_until_approved(): assert st.status == "holding_gate" orch.run_until_gate() assert st.status == "holding_gate" - assert not st.is_done("bug_bash", "bash_done") + assert not st.is_done("bug_bash", "bugbash_complete") def test_approve_advances(): st, orch = _orch() _advance_to_first_gate(orch) orch.approve_gate("ok") - assert st.is_done("bug_bash", "bash_done") + assert st.is_done("bug_bash", "bugbash_complete") orch.run_until_gate() - assert st.current_step == "gate_watch" # next stop after bash_done: the Phase-4 finalize gate + assert st.current_step == "gate_watch" # next stop after bugbash_complete: the Phase-4 finalize gate assert st.status == "holding_gate" @@ -705,11 +706,11 @@ def test_persistence_roundtrip(): # reload — simulates resuming next day st2 = ReleaseState.load(path) assert st2.status == "holding_gate" - assert st2.current_step == "bash_done" + assert st2.current_step == "bugbash_complete" assert st2.readiness_signed # readiness survives the roundtrip orch2 = Orchestrator(CONFIG, st2) orch2.approve_gate("resumed") - assert st2.is_done("bug_bash", "bash_done") + assert st2.is_done("bug_bash", "bugbash_complete") def test_conditional_hotfix_excluded_by_default(): @@ -729,18 +730,18 @@ def test_conditional_hotfix_excluded_by_default(): def test_skip_requires_reason(): st, orch = _orch() - _advance_to_first_gate(orch) # holds at bash_done - act = orch.skip_step("bug_bash", "bash_done", "") # no reason + _advance_to_first_gate(orch) # holds at bugbash_complete + act = orch.skip_step("bug_bash", "bugbash_complete", "") # no reason assert act.kind == "idle" - assert not st.is_done("bug_bash", "bash_done") # unchanged + assert not st.is_done("bug_bash", "bugbash_complete") # unchanged def test_skip_advances_past_gate(): st, orch = _orch() _advance_to_first_gate(orch) - orch.skip_step("bug_bash", "bash_done", "n/a this release") - assert st.is_done("bug_bash", "bash_done") # skipped counts as done - rec = st.steps[st.key("bug_bash", "bash_done")] + orch.skip_step("bug_bash", "bugbash_complete", "n/a this release") + assert st.is_done("bug_bash", "bugbash_complete") # skipped counts as done + rec = st.steps[st.key("bug_bash", "bugbash_complete")] assert rec["status"] == "skipped" orch.run_until_gate() assert st.current_step == "gate_watch" # advanced past the gate to the Phase-4 gate @@ -749,11 +750,11 @@ def test_skip_advances_past_gate(): def test_reopen_step(): st, orch = _orch() _advance_to_first_gate(orch); orch.approve_gate("ok") - assert st.is_done("bug_bash", "bash_done") - orch.reopen_step("bug_bash", "bash_done") - assert not st.is_done("bug_bash", "bash_done") # back to pending + assert st.is_done("bug_bash", "bugbash_complete") + orch.reopen_step("bug_bash", "bugbash_complete") + assert not st.is_done("bug_bash", "bugbash_complete") # back to pending orch.run_until_gate() - assert st.current_step == "bash_done" # gate re-holds + assert st.current_step == "bugbash_complete" # gate re-holds def test_rc_retriggered_reopens_phase2_rc_steps(): @@ -1022,7 +1023,7 @@ def test_only_frontier_phase_shows_current_despite_stale_downstream_progress(): for pid in ("preflight", "ccd"): for s in next(p for p in orch.config["phases"] if p["id"] == pid)["steps"]: orch.state.set_step(pid, s["id"], StepState(status="done", by="test")) - for sid in ("clone_plans_broker", "coordinate"): + for sid in ("clone_plans_broker", "bugbash_updates"): orch.state.set_step("bug_bash", sid, StepState(status="done", by="test")) phases = {p["id"]: p for p in orch.status_report()["phases"]} assert phases["build_verify"]["state"] == "current" and phases["build_verify"]["current"] @@ -1151,7 +1152,7 @@ def test_notify_digest_reports_gate_and_progress(): msg = render.notification(orch.status_report()) assert "Progress:" in msg assert "Action needed now" in msg # ui_failures is the live hold - assert "your approval" in msg # the bash_done gate is listed among the human touchpoints + assert "your approval" in msg # the bugbash_complete gate is listed among the human touchpoints def test_notify_json_carries_owner_and_subject(): @@ -1392,7 +1393,7 @@ def test_registry_relocates_release_automations_into_release_folder(): import os as _os, json as _json with tempfile.TemporaryDirectory() as tmp: reg = AutomationRegistry(tmp, release="2026-08") - reg.register("a2", "Phase-3 watcher", release="2026-08", steps=["bug_bash.bash_done"]) + reg.register("a2", "Phase-3 watcher", release="2026-08", steps=["bug_bash.bugbash_complete"]) reg.register("sh", "Release push reminders", shared=True, purpose="push") rel_file = _os.path.join(tmp, "2026-08", "_automations.json") shared_file = _os.path.join(tmp, "_automations.json") @@ -2993,6 +2994,149 @@ def test_record_bugbash_chat_stores_id_and_marks_done(): assert after.get_step("bug_bash", "activate_chat").status == "blocked" +# ---- Phase 3: bugbash_updates (periodic poster) ---- + +def test_bugbash_holidays_and_working_window(): + """US federal holidays are hardcoded (with observed shifts); is_working_time is a + weekday, not-a-holiday, 09:00–18:00 gate.""" + from datetime import datetime + from tools import bugbash as BB + h = BB.us_holidays(2026) + assert BB.date(2026, 7, 3) in h # July 4 (Sat) observed Fri Jul 3 + assert BB.date(2026, 11, 26) in h # Thanksgiving (4th Thu) + assert BB.date(2026, 12, 25) in h # Christmas + assert BB.is_working_time(datetime(2026, 8, 21, 10)) # Fri 10am + assert not BB.is_working_time(datetime(2026, 8, 21, 8)) # before 9 + assert not BB.is_working_time(datetime(2026, 8, 21, 18)) # 18:00 = closed + assert not BB.is_working_time(datetime(2026, 8, 22, 10)) # Saturday + assert not BB.is_working_time(datetime(2026, 7, 3, 10)) # observed holiday + + +def test_bugbash_render_mentions_finished_and_complete(): + """render_update @mentions (with ) only owners with remaining tests; lists + not-run + failed + blocked (failed first) each with a link; finished owners appear by + name with an 'all completed' line; all_complete flips when 0 remain.""" + from tools import bugbash as BB + prog = {"total": 4, "done": 2, "remaining": 2, "unassigned": 0, "owners": { + "a@x": {"name": "Alice", "total": 2, "done": 0, "remaining": 2, "tests": [ + {"id": "101", "name": "Login", "url": "u101", "state": "notrun"}, + {"id": "102", "name": "MFA", "url": "u102", "state": "failed"}]}, + "b@x": {"name": "Bob", "total": 2, "done": 2, "remaining": 0, "tests": [ + {"id": "201", "name": "X", "url": "u", "state": "passed"}]}}} + html, mentions = BB.render_update(prog, "August 2026", + [{"name": "Broker plan", "url": "bp"}]) + assert mentions == [{"id": 0, "upn": "a@x", "name": "Alice"}] # only Alice (remaining) + assert 'Alice' in html # id matches mentions[0] + # the FAILED test is surfaced (Option A) with a link, and ordered before the not-run one + assert '102' in html and "(Failed)" in html + assert html.index("102") < html.index("101") # failed listed first + assert "all 2 tests completed" in html and "=1).""" + d = date(year, month, 1) + offset = (weekday - d.weekday()) % 7 + return d + timedelta(days=offset + 7 * (n - 1)) + + +def _last_weekday(year, month, weekday): + """The date of the LAST `weekday` of `month`.""" + if month == 12: + nxt = date(year + 1, 1, 1) + else: + nxt = date(year, month + 1, 1) + d = nxt - timedelta(days=1) + return d - timedelta(days=(d.weekday() - weekday) % 7) + + +def _observed(d): + """Federal-holiday observation: Sat -> Fri, Sun -> Mon.""" + if d.weekday() == 5: + return d - timedelta(days=1) + if d.weekday() == 6: + return d + timedelta(days=1) + return d + + +def us_holidays(year): + """Set of observed US FEDERAL holiday dates for `year` (hardcoded rules).""" + h = { + _observed(date(year, 1, 1)), # New Year's Day + _nth_weekday(year, 1, 0, 3), # MLK Jr. — 3rd Mon Jan + _nth_weekday(year, 2, 0, 3), # Presidents' Day — 3rd Mon Feb + _last_weekday(year, 5, 0), # Memorial Day — last Mon May + _observed(date(year, 6, 19)), # Juneteenth + _observed(date(year, 7, 4)), # Independence Day + _nth_weekday(year, 9, 0, 1), # Labor Day — 1st Mon Sep + _nth_weekday(year, 10, 0, 2), # Columbus/Indigenous — 2nd Mon Oct + _observed(date(year, 11, 11)), # Veterans Day + _nth_weekday(year, 11, 3, 4), # Thanksgiving — 4th Thu Nov + _observed(date(year, 12, 25)), # Christmas + } + return h + + +def is_holiday(d): + return d in us_holidays(d.year) + + +def is_working_time(now: datetime) -> bool: + """True if `now` (local) is a weekday, not a US holiday, and within 09:00–18:00.""" + d = now.date() + if now.weekday() >= 5 or is_holiday(d): + return False + return WORK_START_HOUR <= now.hour < WORK_END_HOUR + + +# ----------------------------------------------------------------- progress + +# Per-CASE verdict from its point outcomes (a case may have several points/configs). +_DONE_OUTCOMES = {"passed", "failed", "blocked", "notapplicable"} + + +def _case_state(outcomes): + """Aggregate a case's point outcomes (lowercased) into one display state key: + failed > blocked > passed > na > notrun (any not-yet-run point => notrun).""" + o = {str(x).lower() for x in outcomes} + if not o or (o - _DONE_OUTCOMES): # any unspecified/ready/none -> not run yet + if "failed" in o: + return "failed" # a fail already recorded still shows red + return "notrun" + if "failed" in o: + return "failed" + if "blocked" in o: + return "blocked" + if o <= {"notapplicable"}: + return "na" + return "passed" + + +_STATE_ICON = {"passed": "✅", "failed": "❌", "blocked": "⛔", "na": "➖", "notrun": "⬜"} +_STATE_WORD = {"passed": "Passed", "failed": "Failed", "blocked": "Blocked", + "na": "N/A", "notrun": "Not run"} + + +def _points_of_suite(plan_id, suite_id, timeout=90): + """(ok, [{case_id, name, outcome}], detail) — the test points of ONE suite.""" + url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/Suites/{suite_id}" + f"/TestPoint?api-version=7.1") + ok, items, d = P._ado_rest_get_all(url, timeout) + if not ok: + return (False, None, d) + out = [] + for pt in items: + tc = pt.get("testCaseReference") or {} + res = pt.get("results") or {} + out.append({"case_id": str(tc.get("id")), "name": tc.get("name"), + "outcome": res.get("outcome") or pt.get("outcome") or "unspecified"}) + return (True, out, "") + + +def _broker_points(broker_plan_id, suite_name, timeout=90): + """Points across the Broker 'Manual Tests (Android Broker)' subtree of the plan.""" + okr, root, d = D.find_suite_id_by_name(broker_plan_id, suite_name, timeout) + if not okr or not root: + return (False, None, d or f"'{suite_name}' suite not found in plan {broker_plan_id}") + ok, subtree, d = D._suite_subtree(broker_plan_id, root, timeout) + if not ok: + return (False, None, d) + pts = [] + for sid in subtree: + okp, sp, dp = _points_of_suite(broker_plan_id, sid, timeout) + if not okp: + return (False, None, dp) + pts += sp + return (True, pts, "") + + +def gather_progress(broker_plan_id, broker_suite_name, auth_plan_id, auth_suite_id, + timeout=90): + """(ok, progress, detail). Reads live test points from the Broker manual subtree + + the Auth bug-bash suite, groups the CASES by their System.AssignedTo, and computes a + per-owner + overall breakdown. + + progress = { + total, done, remaining, + owners: { upn: {name, total, done, remaining, + tests: [{id, name, url, state}]} }, + unassigned: } + """ + okb, bpts, db = _broker_points(broker_plan_id, broker_suite_name, timeout) + if not okb: + return (False, None, f"broker: {db}") + oka, apts, da = _points_of_suite(auth_plan_id, auth_suite_id, timeout) + if not oka: + return (False, None, f"auth: {da}") + + # case_id -> {name, outcomes:[...]} + cases = {} + for pt in bpts + apts: + cid = pt["case_id"] + c = cases.setdefault(cid, {"name": pt.get("name"), "outcomes": []}) + c["outcomes"].append(pt.get("outcome")) + + # owner (AssignedTo) per case + oka2, amap, da2 = D._cases_assignedto(list(cases.keys()), timeout) + if not oka2: + return (False, None, f"assignedTo: {da2}") + + owners, unassigned = {}, 0 + total = done = 0 + for cid, c in cases.items(): + total += 1 + state = _case_state(c["outcomes"]) + is_done = state in ("passed", "na") # only clean-pass / N-A count as done; + if is_done: # failed + blocked + notrun are "remaining" + done += 1 + upn = amap.get(cid) + if not upn: + unassigned += 1 + continue + o = owners.setdefault(upn, {"name": upn, "total": 0, "done": 0, "remaining": 0, + "tests": []}) + o["total"] += 1 + o["done"] += 1 if is_done else 0 + o["remaining"] += 0 if is_done else 1 + o["tests"].append({"id": cid, "name": c.get("name") or f"Test {cid}", + "url": f"{ORG}/{PROJECT}/_workitems/edit/{cid}", "state": state}) + + return (True, {"total": total, "done": done, "remaining": total - done, + "owners": owners, "unassigned": unassigned}, "") + + +# ----------------------------------------------------------------- render + +def all_complete(progress) -> bool: + return bool(progress) and progress.get("total", 0) > 0 and progress.get("remaining", 0) == 0 + + +def render_update(progress, month_year, plan_links, name_by_upn=None): + """(html, mentions) for the Teams chat update. + + mentions = [{upn, name}] — the owners who still have REMAINING tests (they get an + @mention). Owners who finished all appear by NAME with an 'all completed' line and are + NOT mentioned. `plan_links` = [{name,url}] (the two live test plans). + """ + name_by_upn = name_by_upn or {} + + def disp(upn, fallback): + return name_by_upn.get(upn) or fallback + + total, done = progress.get("total", 0), progress.get("done", 0) + pct = round(done * 100.0 / total) if total else 0 + owners = progress.get("owners") or {} + mentions = [] + + rows = [] + # remaining-first, most-remaining at the top + for upn in sorted(owners, key=lambda u: (-owners[u]["remaining"], disp(u, u).lower())): + o = owners[upn] + who = disp(upn, o["name"]) + if o["remaining"] == 0: + rows.append(f'
    {who} — ' + f'all {o["total"]} tests completed ✅
    ') + continue + mi = len(mentions) + mentions.append({"id": mi, "upn": upn, "name": who}) + # list every not-done test (failed → blocked → not-run), each with its state icon + _ORDER = {"failed": 0, "blocked": 1, "notrun": 2} + pending = sorted((t for t in o["tests"] if t["state"] in _ORDER), + key=lambda t: (_ORDER[t["state"]], t["id"])) + items = "".join( + f'
  • {_STATE_ICON.get(t["state"], "⬜")} ' + f'{t["id"]} — {t["name"]} ' + f'({_STATE_WORD.get(t["state"], t["state"])})
    • ' + for t in pending) + rows.append( + f'
    {who} — ' + f'{o["done"]}/{o["total"]} done, {o["remaining"]} remaining:' + f'
      {items}
    ') + + links = "  ·  ".join(f'{l["name"]}' for l in (plan_links or [])) + html = ( + f'
    ' + f'

    🐞 {month_year} Bug Bash — progress update
    ' + f'{done}/{total} tests done ({pct}%) · {progress.get("remaining",0)} remaining. ' + f'Mark pass/fail in the ADO test plan; report bugs/logs here.

    ' + f'

    {links}

    ' + f'{"".join(rows)}' + f'
    ') + return html, mentions From 4b6442ead462511e587c0de3b810761e389968b4 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Sat, 22 Aug 2026 06:14:01 +0100 Subject: [PATCH 87/90] Phase 3 bug bash: flat Broker monthly test plan Replace the full-hierarchy Copy Test Plan with a flat build: create a fresh plan + one static suite "Manual Tests (Android Broker)" holding just the resolved manual-broker cases, each pinned to the two flight configs (ECS + LocalFlights). Only the relevant tests, flat and trackable; distribute_tests and the progress updates find the suite by name so it is a drop-in. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 34 +++-- .../steps/bug_bash/clone_plans_broker.py | 50 +++--- release-agent/tests/test_engine.py | 104 ++++++++----- release-agent/tools/testplans.py | 144 ++++++++++-------- 4 files changed, 191 insertions(+), 141 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 3c202eb6..7226ae45 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -339,26 +339,36 @@ build_verify.rc_report: # =============================== Phase 3 · bug_bash =============================== bug_bash.clone_plans_broker: - summary: "Copy the Broker master test plan to a new plan for this release." - what: > - Each release COPIES the Broker master template (ADO test plan #2007357, area - 'Engineering\\Auth Client\\Broker\\Android') to a new plan named - 'Android Monthly Release - ' (e.g. 'Android Monthly Release - Aug 2026'), - REFERENCING the existing test cases (the ADO clone default — it shares test cases, - it does not duplicate them). This is the Broker half of the old clone_plans step. - who: "Scout runs it automatically during `next` (an agent step, via the ADO test-plan clone API)." + summary: "Build this release's FLAT Broker test plan (one suite of the manual-broker cases)." + what: > + Each release gets a FLAT Broker test plan named 'Android Monthly Release - ' + (e.g. 'Android Monthly Release - Aug 2026', area 'Engineering\\Auth Client\\Broker\\ + Android'). Instead of ADO's "Copy Test Plan" (which reproduces the master's whole + 45-suite tree), Scout creates a fresh plan with ONE static suite + 'Manual Tests (Android Broker)' holding just the manual-broker cases resolved from the + master template (#2007357, subtree suite 2008656) — the only tests the bug bash runs. + Each case is pinned to the two flight configurations (293 'RC MSAL - RC Broker' / ECS + + 330 '… (LocalFlights)'), giving 2 points/case, and REFERENCES the existing test cases + (shared, not duplicated). Flat = easy to track; distribute_tests + the progress updates + find the suite by name, so it's a drop-in. + who: "Scout runs it automatically during `next` (an agent step, via the ADO test-plan API)." how: > Idempotent — the new plan id is stashed on the step; a re-run re-confirms that plan - exists and reports done without cloning again. If the clone fails (auth/API), the - step blocks with the ADO error; fix and `next`, or `skip --reason` to override. + exists and reports done without rebuilding. If the build fails (auth/API) the half-built + plan is deleted and the step blocks with the ADO error; fix and `next`, or `skip + --reason` to override. links: - - name: "Broker master test plan (#2007357)" + - name: "Broker master test plan (#2007357, Manual Tests subtree)" url: "https://identitydivision.visualstudio.com/Engineering/_testPlans/define?planId=2007357&suiteId=2008656" - name: "Broker test-plans doc" url: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/internal-release-checklist/test-plans" faqs: + - q: "Why flat instead of copying the whole master plan?" + a: "The bug bash only runs the 'Manual Tests (Android Broker)' subtree, and the master's nested + dynamic suites are hard to track. Flattening to one static suite gives a single, trackable list of exactly the relevant cases." - q: "Does it duplicate the test cases?" - a: "No — it references the existing test cases (the 'Reference existing test cases' clone option). Edit test cases only in the master template." + a: "No — it references the existing test cases (shared). Edit test cases only in the master template." + - q: "Does it auto-update if the master's queries change later?" + a: "No — flattening resolves the master's dynamic suites to a frozen snapshot at build time, which is what you want for a stable monthly bug-bash cut. Rebuild (delete the plan id + re-run) to re-snapshot." bug_bash.clone_plans_auth: summary: "Create the Authenticator bug-bash query-based test suite for this release." diff --git a/release-agent/steps/bug_bash/clone_plans_broker.py b/release-agent/steps/bug_bash/clone_plans_broker.py index 0ef79de4..0e3a39e7 100644 --- a/release-agent/steps/bug_bash/clone_plans_broker.py +++ b/release-agent/steps/bug_bash/clone_plans_broker.py @@ -1,18 +1,19 @@ -"""Step: `clone_plans_broker` — copy the Broker master test plan for this release -(Phase 3, bug_bash). +"""Step: `clone_plans_broker` — build this release's FLAT Broker test plan (Phase 3, bug_bash). -Per the broker release doc, each release COPIES the master template plan (2007357) -to a new plan "Android Monthly Release - ", referencing the existing test -cases (the ADO clone default — shares test cases, doesn't duplicate them). This is the -Broker half of the old `clone_plans` stub. +Instead of ADO's "Copy Test Plan" (which reproduces the master's whole 45-suite tree), this +creates a fresh plan "Android Monthly Release - " with ONE static suite +"Manual Tests (Android Broker)" holding just the relevant manual-broker cases (the only tests +the bug bash runs), each pinned to the two flight configs (ECS + LocalFlights). Cases are +referenced (shared, not duplicated). Flat = easy to track; downstream steps find the suite by +name so it's a drop-in. -Idempotent: the created plan id is stashed on the step (data.plan_id). On a re-run the -step re-confirms that plan still exists and reports done WITHOUT cloning again — so a -`next` that re-enters Phase 3 never spawns a duplicate plan. +Idempotent: the created plan id is stashed on the step (data.plan_id). On a re-run the step +re-confirms that plan still exists and reports done WITHOUT rebuilding — so a `next` that +re-enters Phase 3 never spawns a duplicate plan. Mock knobs (mocks.local.yaml / tests): - plan_id : pretend the clone already ran (this plan id) — verifies + reports done. - clone_id : the id the clone POST should "return" (skip the live CloneOperation). + plan_id : pretend the build already ran (this plan id) — verifies + reports done. + clone_id : the id the create should "return" (skip the live plan build). fail : a detail string → force a Blocked (simulate an API/auth failure). """ from __future__ import annotations @@ -27,8 +28,8 @@ MOCKABLE = { "name": {"kind": "input", "desc": "Override the destination plan name (e.g. a 'TEST ...' name for a safe live run)."}, - "plan_id": {"kind": "input", "desc": "Pretend the clone already produced this plan id (idempotency test)."}, - "clone_id": {"kind": "input", "desc": "Id the clone should return (skip the live CloneOperation)."}, + "plan_id": {"kind": "input", "desc": "Pretend the build already produced this plan id (idempotency test)."}, + "clone_id": {"kind": "input", "desc": "Id the create should return (skip the live plan build)."}, "fail": {"kind": "input", "desc": "Force a Blocked with this detail (simulate an API failure)."}, } @@ -47,37 +48,38 @@ def build(state): dest = T.broker_plan_name(state.release_id) step = state.get_step("bug_bash", ID) - # Already cloned? A test injects `plan_id` to assert idempotency; otherwise the - # stored id from a prior run is re-confirmed against ADO. Either way → done, no re-clone. + # Already built? A test injects `plan_id` to assert idempotency; otherwise the stored id + # from a prior run is re-confirmed against ADO. Either way → done, no rebuild. injected = mock_input("plan_id", MISSING) if injected is not MISSING: - return Done(f"Broker test plan already cloned for {state.release_id}: " + return Done(f"Broker test plan already built for {state.release_id}: " f"'{dest}' (plan {injected}).", links=_links(injected)) stored = (step.data or {}).get("plan_id") if stored: ok, info, _ = T.get_plan(stored) if ok and info: - return Done(f"Broker test plan already cloned for {state.release_id}: " + return Done(f"Broker test plan already built for {state.release_id}: " f"'{info.get('name') or dest}' (plan {stored}).", links=_links(stored)) - # recorded id no longer resolves — fall through and re-clone + # recorded id no longer resolves — fall through and rebuild - # Clone the master (or take the injected clone_id offline). + # Build the flat plan (or take the injected clone_id offline). clone_id = mock_input("clone_id", MISSING) if clone_id is MISSING: - ok, clone_id, detail = T.clone_broker_plan(dest) + ok, clone_id, detail = T.create_broker_flat_plan(dest) if not ok: hint = " — run `az login`" if str(detail).startswith("AUTH") else "" return Blocked( - f"clone_plans_broker: could not copy the master test plan " - f"(#{T.BROKER_MASTER_PLAN}) to '{dest}' ({detail}){hint}.") + f"clone_plans_broker: could not build the flat Broker test plan '{dest}' " + f"from the master (#{T.BROKER_MASTER_PLAN}) ({detail}){hint}.") step.data = dict(step.data or {}) step.data["plan_id"] = clone_id step.data["plan_name"] = dest state.set_step("bug_bash", ID, step) return Done( - f"Copied the Broker master test plan (#{T.BROKER_MASTER_PLAN}) to '{dest}' " - f"(plan {clone_id}), referencing existing test cases.", links=_links(clone_id)) + f"Built the flat Broker test plan '{dest}' (plan {clone_id}) — one " + f"'{T.BROKER_MANUAL_SUITE_NAME}' suite of the manual-broker cases, referencing " + f"existing test cases.", links=_links(clone_id)) run = legacy_run(build) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index f883c14f..5a0cec14 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2737,17 +2737,17 @@ def test_testplans_names_and_query(): assert "contains 'Android'" in q and "contains 'ReleaseBugBash'" in q and "Identity Apps" in q -def test_clone_plans_broker_clones_then_idempotent(): - """First run copies the master plan and stashes the new plan id; a re-run with that id - stored reports done WITHOUT cloning again.""" +def test_clone_plans_broker_builds_flat_then_idempotent(): + """First run builds the flat plan and stashes the new plan id; a re-run with that id + stored reports done WITHOUT rebuilding.""" st, out = _bb_build("clone_plans_broker", {"clone_id": "5551212"}) assert out["kind"] == "done" - assert "Copied the Broker master test plan" in out["note"] and "5551212" in out["note"] + assert "flat Broker test plan" in out["note"] and "5551212" in out["note"] assert st.get_step("bug_bash", "clone_plans_broker").data["plan_id"] == "5551212" assert out["links"][0]["url"].endswith("planId=5551212") - # idempotent: an injected existing plan id → already-cloned, no re-clone + # idempotent: an injected existing plan id → already-built, no rebuild _, out2 = _bb_build("clone_plans_broker", {"plan_id": "5551212"}) - assert out2["kind"] == "done" and "already cloned" in out2["note"] + assert out2["kind"] == "done" and "already built" in out2["note"] def test_clone_plans_broker_blocks_on_api_failure(): @@ -3147,52 +3147,72 @@ def test_clone_plans_name_override_knob(): assert "TEST Android/release/08/2026" in out2["note"] -def test_clone_broker_plan_waits_for_async_completion(): - """clone_broker_plan POSTs the CloneOperation then POLLS the op until 'succeeded' - before returning — so the plan is fully populated when the step reports done (the - async 'seems empty if read too early' gap). state lives under cloneOperationResponse.""" +def test_create_broker_flat_plan_builds_single_suite_with_configs(): + """create_broker_flat_plan: POST plan → POST one static suite (pinned to the 2 flight + configs, no inherited defaults) → add the resolved manual cases with both config point + assignments. Returns the new plan id.""" from tools import pipelines as P from tools import testplans as T - post = (True, {"destinationTestPlan": {"id": 9001}, - "cloneOperationResponse": {"opId": 77, "state": "queued"}}, "") - # op poll: inProgress, then inProgress, then succeeded - op_pages = [ - (True, {"cloneOperationResponse": {"state": "inProgress"}}, {}, ""), - (True, {"cloneOperationResponse": {"state": "inProgress"}}, {}, ""), - (True, {"cloneOperationResponse": {"state": "succeeded"}}, {}, ""), - ] - calls = {"get": 0} + from tools import distribution as D + sends = [] def fake_send(url, method, body, timeout): - return post - - def fake_get_h(url, timeout): - i = calls["get"]; calls["get"] += 1 - return op_pages[min(i, len(op_pages) - 1)] - - o_send, o_get = P._ado_rest_send, P._ado_rest_get_h - P._ado_rest_send, P._ado_rest_get_h = fake_send, fake_get_h + sends.append((url, method, body)) + if url.endswith(f"plans?{T._API}"): # create plan + return (True, {"id": 9100, "rootSuite": {"id": 9101}}, "") + if "/suites?" in url: # create suite + return (True, {"id": 9102}, "") + if "/TestCase?" in url: # add cases + return (True, {"value": body}, "") + return (True, {}, "") + + o_send, o_cases = P._ado_rest_send, D.broker_manual_cases + P._ado_rest_send = fake_send + D.broker_manual_cases = lambda *a, **k: (True, [{"id": "111", "assignee": "a@x"}, + {"id": "222", "assignee": "b@x"}], "") try: - ok, pid, d = T.clone_broker_plan("TEST plan", poll_secs=0) + ok, pid, d = T.create_broker_flat_plan("TEST FLAT plan") finally: - P._ado_rest_send, P._ado_rest_get_h = o_send, o_get - assert ok and pid == 9001 and d == "" - assert calls["get"] >= 3 # polled until 'succeeded' - - -def test_clone_broker_plan_blocks_on_failed_clone_op(): - """A failed clone operation surfaces as (False, ...) so the step blocks.""" + P._ado_rest_send, D.broker_manual_cases = o_send, o_cases + assert ok and pid == 9100 and d == "" + # suite create pinned to the 2 configs, no inherited defaults + suite_body = next(b for (u, m, b) in sends if "/suites?" in u) + assert suite_body["name"] == T.BROKER_MANUAL_SUITE_NAME + assert suite_body["inheritDefaultConfigurations"] is False + assert [c["id"] for c in suite_body["defaultConfigurations"]] == T.BROKER_CONFIGS + # each case added with a point assignment per config + add_body = next(b for (u, m, b) in sends if "/TestCase?" in u) + assert len(add_body) == 2 + assert [pa["configurationId"] for pa in add_body[0]["pointAssignments"]] == T.BROKER_CONFIGS + + +def test_create_broker_flat_plan_cleans_up_on_case_failure(): + """If adding cases fails after the plan is created, the half-built plan is DELETEd so a + re-run starts clean, and the function returns (False, ...).""" from tools import pipelines as P from tools import testplans as T - o_send, o_get = P._ado_rest_send, P._ado_rest_get_h - P._ado_rest_send = lambda *a, **k: (True, {"destinationTestPlan": {"id": 1}, - "cloneOperationResponse": {"opId": 5, "state": "queued"}}, "") - P._ado_rest_get_h = lambda *a, **k: (True, {"cloneOperationResponse": {"state": "failed"}}, {}, "") + from tools import distribution as D + calls = [] + + def fake_send(url, method, body, timeout): + calls.append((method, url)) + if url.endswith(f"plans?{T._API}"): + return (True, {"id": 9200, "rootSuite": {"id": 9201}}, "") + if "/suites?" in url: + return (True, {"id": 9202}, "") + if "/TestCase?" in url: + return (False, None, "HTTP 400: bad point assignment") + return (True, {}, "") + + o_send, o_cases = P._ado_rest_send, D.broker_manual_cases + P._ado_rest_send = fake_send + D.broker_manual_cases = lambda *a, **k: (True, [{"id": "111", "assignee": "a@x"}], "") try: - ok, pid, d = T.clone_broker_plan("TEST plan", poll_secs=0) + ok, pid, d = T.create_broker_flat_plan("TEST FLAT plan") finally: - P._ado_rest_send, P._ado_rest_get_h = o_send, o_get - assert not ok and "failed" in d + P._ado_rest_send, D.broker_manual_cases = o_send, o_cases + assert not ok and "400" in d + assert any(m == "DELETE" and "/plans/9200" in u for (m, u) in calls) # cleaned up def test_ado_rest_get_all_follows_header_continuation_token(): diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 8fc7c1e6..f7012733 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -2,11 +2,13 @@ Two DIFFERENT release procedures, per the team docs: - * BROKER — COPY the master test plan (a real ADO "Copy Test Plan" / CloneOperation). - Master: plan 2007357 / rootSuite 2007358 ("Android Monthly Release Master Test - Plan", area 'Engineering\\Auth Client\\Broker\\Android'). Each release clones it to - a new plan "Android Monthly Release - ", REFERENCING the existing test - cases (the ADO clone default — it shares test cases, doesn't duplicate them). + * BROKER — build a FLAT monthly plan. Instead of ADO's "Copy Test Plan" (which + reproduces the master's whole 45-suite tree), create a fresh plan "Android Monthly + Release - " with ONE static suite "Manual Tests (Android Broker)" holding + just the manual-broker cases resolved from the master's subtree (2007357 / suite + 2008656), each pinned to the two flight configs (293 ECS + 330 LocalFlights). Cases are + REFERENCED (shared, not duplicated). Flat = trackable; downstream steps find the suite + by name so it's a drop-in. Doc: eng.ms/.../internal-release-checklist/test-plans * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing @@ -28,9 +30,19 @@ PROJECT = P.ENGINEERING_PROJECT # Engineering _API = "api-version=7.1" -# ---- Broker: master template to clone ---- +# ---- Broker: master template + the flat monthly copy ---- BROKER_MASTER_PLAN = 2007357 BROKER_MASTER_ROOT_SUITE = 2007358 +# The "Manual Tests (Android Broker)" subtree of the master — the ONLY tests the release +# bug bash runs. The monthly copy flattens this subtree into a single static suite. +BROKER_MANUAL_ROOT_SUITE = 2008656 +BROKER_MANUAL_SUITE_NAME = "Manual Tests (Android Broker)" +# Test configurations the bug bash runs each case under (the two flight pipelines): +# 293 = "RC MSAL - RC Broker" (ECS flights) +# 330 = "RC MSAL - RC Broker (LocalFlights)" (Local flights) +# Assigned explicitly so the flat plan gets exactly 2 points/case (matches the master's +# matrix) instead of inheriting the project's ~190 default configurations. +BROKER_CONFIGS = [293, 330] BROKER_AREA_PATH = "Engineering\\Auth Client\\Broker\\Android" BROKER_ITERATION = "Engineering" @@ -130,66 +142,72 @@ def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): # ---------------------------------------------------------------- writes -def _clone_op(op_field, *keys): - """Read a field from a CloneOperation response, which nests the live status under - `cloneOperationResponse` (state/opId/completionDate/cloneStatistics) but the ids at - the top level. Tries the nested object first, then the top level.""" - nested = (op_field or {}).get("cloneOperationResponse") or {} - for k in keys: - if nested.get(k) is not None: - return nested.get(k) - if (op_field or {}).get(k) is not None: - return op_field.get(k) - return None - - -def clone_broker_plan(dest_name, timeout=120, poll_secs=3, max_polls=40): - """COPY the Broker master plan to a new plan `dest_name`, referencing existing test - cases (ADO clone default), and WAIT for the async clone to finish. Returns - (ok, new_plan_id, detail). - - ADO's CloneOperation is asynchronous: the POST returns a destination plan id while the - suite/test-case copy runs in the background. We poll the operation until its state is - 'succeeded' before returning, so the plan is fully populated when the step reports done - (a too-early read would otherwise show an empty/partial plan). Mirrors the doc's - "Copy Test Plan → Reference existing test cases → Create". +def create_broker_flat_plan(dest_name, timeout=120): + """Build the release's FLAT Broker test plan and return (ok, new_plan_id, detail). + + Instead of ADO's "Copy Test Plan" (which reproduces the master's whole 45-suite + hierarchy), this creates a fresh plan with a SINGLE static suite + "Manual Tests (Android Broker)" holding just the relevant manual-broker cases — the + only tests the bug bash runs. Steps: + + 1. create an empty plan `dest_name` (Broker area/iteration); + 2. create one static child suite named "Manual Tests (Android Broker)" pinned to the + two flight configurations (293 ECS + 330 LocalFlights) — inheritDefaultConfigurations + is False so the plan's ~190 project-default configs are NOT applied; + 3. resolve the master's "Manual Tests (Android Broker)" subtree (flattening its dynamic + sub-suites) to the current set of cases, and add them all to the flat suite, + referencing the existing work items (shared, not duplicated), each assigned to both + configs → 2 points/case (matches the master's matrix). + + Downstream (distribute_tests, gather_progress) find the suite by name, so a flat plan is + a drop-in. On a partial failure the half-built plan is best-effort deleted so a re-run + starts clean. """ - import time - url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation?api-version=7.1-preview.2" - body = { - # copy every suite + the hierarchy; do NOT clone requirements. Not setting a - # test-case duplication flag => ADO REFERENCES the existing test cases - # (cloneStatistics.clonedTestCasesCount stays 0 — shared, not duplicated). - # (copyAncestorHierarchy must be true when source suiteIds are given, else ADO 400.) - "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": True, - "cloneRequirements": False, "copyComments": False}, - "destinationTestPlan": {"name": dest_name, "project": PROJECT, - "areaPath": BROKER_AREA_PATH, "iteration": BROKER_ITERATION}, - "sourceTestPlan": {"id": BROKER_MASTER_PLAN, "suiteIds": [BROKER_MASTER_ROOT_SUITE]}, - } - ok, j, d = P._ado_rest_send(url, "POST", body, timeout) + from tools import distribution as D # local import avoids a circular import + + # 1) empty plan + ok, j, d = P._ado_rest_send( + f"{ORG}/{PROJECT}/_apis/testplan/plans?{_API}", "POST", + {"name": dest_name, "area": {"name": BROKER_AREA_PATH}, "iteration": BROKER_ITERATION}, + timeout) if not ok: return (False, None, d) - dest = (j or {}).get("destinationTestPlan") or {} - pid = dest.get("id") - if not pid: - return (False, None, f"clone returned no destination plan id (state={_clone_op(j, 'state')})") - op_id = _clone_op(j, "opId") - - # Poll the operation to completion so the plan is populated before we report done. - if op_id: - op_url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation/{op_id}" - f"?api-version=7.1-preview.2") - for _ in range(max_polls): - ok_o, op, _h, d_o = P._ado_rest_get_h(op_url, 60) - if not ok_o: - break # can't read op → best-effort, return pid - state = str(_clone_op(op, "state") or "").lower() - if state in ("succeeded", "completed"): - break - if state == "failed": - return (False, None, f"clone operation failed (op {op_id})") - time.sleep(poll_secs) + pid = (j or {}).get("id") + root = ((j or {}).get("rootSuite") or {}).get("id") + if not pid or not root: + return (False, None, "plan create returned no id/rootSuite") + + def _cleanup(reason): + P._ado_rest_send(f"{ORG}/{PROJECT}/_apis/testplan/plans/{pid}?{_API}", "DELETE", None, 60) + return (False, None, reason) + + # 2) single flat static suite, pinned to the two flight configs + ok2, sj, d2 = P._ado_rest_send( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{pid}/suites?api-version=7.1-preview.1", "POST", + {"suiteType": "staticTestSuite", "name": BROKER_MANUAL_SUITE_NAME, + "parentSuite": {"id": root}, "inheritDefaultConfigurations": False, + "defaultConfigurations": [{"id": c} for c in BROKER_CONFIGS]}, + timeout) + if not ok2: + return _cleanup(f"suite create failed: {d2}") + suite_id = (sj or {}).get("id") + if not suite_id: + return _cleanup("suite create returned no id") + + # 3) resolve the master's manual-broker cases and add them to the flat suite + okc, cases, dc = D.broker_manual_cases(BROKER_MASTER_PLAN, BROKER_MANUAL_ROOT_SUITE, timeout) + if not okc: + return _cleanup(f"could not resolve master manual cases: {dc}") + if not cases: + return _cleanup("master 'Manual Tests (Android Broker)' subtree resolved to 0 cases") + body = [{"workItem": {"id": int(c["id"])}, + "pointAssignments": [{"configurationId": cid} for cid in BROKER_CONFIGS]} + for c in cases] + oka, _aj, da = P._ado_rest_send( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{pid}/Suites/{suite_id}/TestCase" + f"?api-version=7.1-preview.3", "POST", body, max(timeout, 120)) + if not oka: + return _cleanup(f"adding {len(cases)} cases failed: {da}") return (True, pid, "") From 0b0ca2a9352b9662fa71ad42aae6401a6aee3482 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Sat, 22 Aug 2026 06:37:43 +0100 Subject: [PATCH 88/90] Phase 3 bug bash: add Native Auth folder to Broker test plan build_broker_plan now creates two top-level folders: the flat "Manual Tests (Android Broker)" suite plus "Manual Tests (Native Auth)" copied as a folder (its dynamic query suite preserved, pinned to ECS-only config). Adds reusable suite/case helpers and a recursive subtree replicator. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 33 ++-- release-agent/config/phases.yaml | 2 +- .../steps/bug_bash/clone_plans_broker.py | 27 +-- release-agent/tests/test_engine.py | 91 +++++---- release-agent/tools/testplans.py | 175 +++++++++++++----- 5 files changed, 220 insertions(+), 108 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 7226ae45..678cb387 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -339,18 +339,19 @@ build_verify.rc_report: # =============================== Phase 3 · bug_bash =============================== bug_bash.clone_plans_broker: - summary: "Build this release's FLAT Broker test plan (one suite of the manual-broker cases)." - what: > - Each release gets a FLAT Broker test plan named 'Android Monthly Release - ' - (e.g. 'Android Monthly Release - Aug 2026', area 'Engineering\\Auth Client\\Broker\\ - Android'). Instead of ADO's "Copy Test Plan" (which reproduces the master's whole - 45-suite tree), Scout creates a fresh plan with ONE static suite - 'Manual Tests (Android Broker)' holding just the manual-broker cases resolved from the - master template (#2007357, subtree suite 2008656) — the only tests the bug bash runs. - Each case is pinned to the two flight configurations (293 'RC MSAL - RC Broker' / ECS + - 330 '… (LocalFlights)'), giving 2 points/case, and REFERENCES the existing test cases - (shared, not duplicated). Flat = easy to track; distribute_tests + the progress updates - find the suite by name, so it's a drop-in. + summary: "Build this release's Broker test plan — a flat Broker suite + the Native Auth folder." + what: > + Each release gets a Broker test plan named 'Android Monthly Release - ' (e.g. + 'Android Monthly Release - Aug 2026', area 'Engineering\\Auth Client\\Broker\\Android'). + Instead of ADO's "Copy Test Plan" (which reproduces the master's whole 45-suite tree), + Scout creates a fresh plan with exactly TWO top-level folders: (1) 'Manual Tests (Android + Broker)' — one FLAT static suite of the manual-broker cases resolved from the master + template (#2007357, subtree suite 2008656), each pinned to the two flight configurations + (293 'RC MSAL - RC Broker' / ECS + 330 '… (LocalFlights)') → 2 points/case; and (2) + 'Manual Tests (Native Auth)' — the master's Native Auth subtree (#2864589) copied AS A + FOLDER (its dynamic query suite preserved, so it stays current), pinned to ECS only (293). + All cases are REFERENCED (shared, not duplicated). distribute_tests + the progress updates + find the Broker suite by name, so it's a drop-in. who: "Scout runs it automatically during `next` (an agent step, via the ADO test-plan API)." how: > Idempotent — the new plan id is stashed on the step; a re-run re-confirms that plan @@ -363,12 +364,12 @@ bug_bash.clone_plans_broker: - name: "Broker test-plans doc" url: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/internal-release-checklist/test-plans" faqs: - - q: "Why flat instead of copying the whole master plan?" - a: "The bug bash only runs the 'Manual Tests (Android Broker)' subtree, and the master's nested + dynamic suites are hard to track. Flattening to one static suite gives a single, trackable list of exactly the relevant cases." + - q: "Why flat Broker but a folder for Native Auth?" + a: "The Broker bug bash runs one manual-tests set that's easiest to track as a single flat list. Native Auth is a self-contained query suite, so it's kept as its own folder (still current, not flattened) — giving exactly two folders in the plan." - q: "Does it duplicate the test cases?" a: "No — it references the existing test cases (shared). Edit test cases only in the master template." - - q: "Does it auto-update if the master's queries change later?" - a: "No — flattening resolves the master's dynamic suites to a frozen snapshot at build time, which is what you want for a stable monthly bug-bash cut. Rebuild (delete the plan id + re-run) to re-snapshot." + - q: "Does it auto-update if the master changes later?" + a: "The Broker suite is a frozen snapshot at build time (stable monthly cut). The Native Auth folder stays dynamic (its query re-resolves), so it reflects the current Native Auth cases. Rebuild (delete the plan id + re-run) to re-snapshot Broker." bug_bash.clone_plans_auth: summary: "Create the Authenticator bug-bash query-based test suite for this release." diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 144a21ef..864c50e7 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -63,7 +63,7 @@ phases: name: "Test / Bug Bash" checklist_phase: 3 steps: - - { id: clone_plans_broker, name: "Clone Broker test plan (copy master template)", owner: agent, maps_to: [T3] } + - { id: clone_plans_broker, name: "Build Broker test plan (flat Broker suite + Native Auth folder)", owner: agent, maps_to: [T3] } - { id: clone_plans_auth, name: "Create Authenticator bug-bash test suite", owner: agent, maps_to: [T3] } - { id: distribute_tests, name: "Distribute bug-bash tests across the team", owner: agent, maps_to: [T2] } - { id: send_invite, name: "Schedule Bug Bash + send Teams invite", owner: agent, source: scout, maps_to: [T2] } diff --git a/release-agent/steps/bug_bash/clone_plans_broker.py b/release-agent/steps/bug_bash/clone_plans_broker.py index 0e3a39e7..08c2891c 100644 --- a/release-agent/steps/bug_bash/clone_plans_broker.py +++ b/release-agent/steps/bug_bash/clone_plans_broker.py @@ -1,11 +1,14 @@ -"""Step: `clone_plans_broker` — build this release's FLAT Broker test plan (Phase 3, bug_bash). +"""Step: `clone_plans_broker` — build this release's Broker test plan (Phase 3, bug_bash). Instead of ADO's "Copy Test Plan" (which reproduces the master's whole 45-suite tree), this -creates a fresh plan "Android Monthly Release - " with ONE static suite -"Manual Tests (Android Broker)" holding just the relevant manual-broker cases (the only tests -the bug bash runs), each pinned to the two flight configs (ECS + LocalFlights). Cases are -referenced (shared, not duplicated). Flat = easy to track; downstream steps find the suite by -name so it's a drop-in. +creates a fresh plan "Android Monthly Release - " with exactly TWO top-level +folders: + • "Manual Tests (Android Broker)" — one FLAT static suite of the manual-broker cases (the + only broker tests the bug bash runs), pinned to the two flight configs (ECS + LocalFlights); + • "Manual Tests (Native Auth)" — copied AS A FOLDER (its dynamic query suite preserved), + pinned to ECS only. +Cases are referenced (shared, not duplicated). Flat Broker = easy to track; Native Auth stays +a dynamic folder so it's current. Downstream steps find the Broker suite by name → drop-in. Idempotent: the created plan id is stashed on the step (data.plan_id). On a re-run the step re-confirms that plan still exists and reports done WITHOUT rebuilding — so a `next` that @@ -62,14 +65,14 @@ def build(state): f"'{info.get('name') or dest}' (plan {stored}).", links=_links(stored)) # recorded id no longer resolves — fall through and rebuild - # Build the flat plan (or take the injected clone_id offline). + # Build the plan (or take the injected clone_id offline). clone_id = mock_input("clone_id", MISSING) if clone_id is MISSING: - ok, clone_id, detail = T.create_broker_flat_plan(dest) + ok, clone_id, detail = T.build_broker_plan(dest) if not ok: hint = " — run `az login`" if str(detail).startswith("AUTH") else "" return Blocked( - f"clone_plans_broker: could not build the flat Broker test plan '{dest}' " + f"clone_plans_broker: could not build the Broker test plan '{dest}' " f"from the master (#{T.BROKER_MASTER_PLAN}) ({detail}){hint}.") step.data = dict(step.data or {}) @@ -77,9 +80,9 @@ def build(state): step.data["plan_name"] = dest state.set_step("bug_bash", ID, step) return Done( - f"Built the flat Broker test plan '{dest}' (plan {clone_id}) — one " - f"'{T.BROKER_MANUAL_SUITE_NAME}' suite of the manual-broker cases, referencing " - f"existing test cases.", links=_links(clone_id)) + f"Built the Broker test plan '{dest}' (plan {clone_id}) — a flat " + f"'{T.BROKER_MANUAL_SUITE_NAME}' suite plus the '{T.BROKER_NATIVE_AUTH_SUITE_NAME}' " + f"folder, referencing existing test cases.", links=_links(clone_id)) run = legacy_run(build) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 5a0cec14..6788c454 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2737,12 +2737,13 @@ def test_testplans_names_and_query(): assert "contains 'Android'" in q and "contains 'ReleaseBugBash'" in q and "Identity Apps" in q -def test_clone_plans_broker_builds_flat_then_idempotent(): - """First run builds the flat plan and stashes the new plan id; a re-run with that id - stored reports done WITHOUT rebuilding.""" +def test_clone_plans_broker_builds_then_idempotent(): + """First run builds the plan (Broker flat suite + Native Auth folder) and stashes the new + plan id; a re-run with that id stored reports done WITHOUT rebuilding.""" st, out = _bb_build("clone_plans_broker", {"clone_id": "5551212"}) assert out["kind"] == "done" - assert "flat Broker test plan" in out["note"] and "5551212" in out["note"] + assert "Broker test plan" in out["note"] and "5551212" in out["note"] + assert "Native Auth" in out["note"] assert st.get_step("bug_bash", "clone_plans_broker").data["plan_id"] == "5551212" assert out["links"][0]["url"].endswith("planId=5551212") # idempotent: an injected existing plan id → already-built, no rebuild @@ -3147,48 +3148,72 @@ def test_clone_plans_name_override_knob(): assert "TEST Android/release/08/2026" in out2["note"] -def test_create_broker_flat_plan_builds_single_suite_with_configs(): - """create_broker_flat_plan: POST plan → POST one static suite (pinned to the 2 flight - configs, no inherited defaults) → add the resolved manual cases with both config point - assignments. Returns the new plan id.""" +def test_build_broker_plan_makes_flat_broker_and_native_auth_folder(): + """build_broker_plan: create plan → flat Broker suite (2 flight configs) + its cases → + Native Auth FOLDER (static parent + the master's dynamic child recreated with its query), + pinned to ECS-only. Returns the new plan id.""" from tools import pipelines as P from tools import testplans as T from tools import distribution as D + NA = T.BROKER_NATIVE_AUTH_ROOT_SUITE + masters = [ + {"id": NA, "name": "Manual Tests (Native Auth)", "suiteType": "staticTestSuite", + "parentSuite": {"id": 2007358}}, + {"id": 5001, "name": "Native Auth Test Manual", "suiteType": "dynamicTestSuite", + "parentSuite": {"id": NA}}, + ] sends = [] + new_suite_id = {"n": 9100} def fake_send(url, method, body, timeout): sends.append((url, method, body)) - if url.endswith(f"plans?{T._API}"): # create plan - return (True, {"id": 9100, "rootSuite": {"id": 9101}}, "") - if "/suites?" in url: # create suite - return (True, {"id": 9102}, "") - if "/TestCase?" in url: # add cases + if url.endswith(f"plans?{T._API}"): + return (True, {"id": 9000, "rootSuite": {"id": 9001}}, "") + if "/suites?" in url: + new_suite_id["n"] += 1 + return (True, {"id": new_suite_id["n"]}, "") + if "/TestCase?" in url: return (True, {"value": body}, "") return (True, {}, "") - o_send, o_cases = P._ado_rest_send, D.broker_manual_cases - P._ado_rest_send = fake_send - D.broker_manual_cases = lambda *a, **k: (True, [{"id": "111", "assignee": "a@x"}, - {"id": "222", "assignee": "b@x"}], "") + def fake_get_all(url, timeout, **k): + if f"/Plans/{T.BROKER_MASTER_PLAN}/suites?" in url: # master suites list + return (True, masters, "") + if "/TestCase?" in url: # static suite's direct cases + return (True, [], "") + return (True, [], "") + + def fake_get_h(url, timeout): + return (True, {"queryString": "SELECT [System.Id] WHERE tag='CIAM'"}, {}, "") + + o = (P._ado_rest_send, P._ado_rest_get_all, P._ado_rest_get_h, D.broker_manual_cases) + P._ado_rest_send, P._ado_rest_get_all, P._ado_rest_get_h = fake_send, fake_get_all, fake_get_h + D.broker_manual_cases = lambda *a, **k: (True, [{"id": "111", "assignee": "a@x"}], "") try: - ok, pid, d = T.create_broker_flat_plan("TEST FLAT plan") + ok, pid, d = T.build_broker_plan("TEST plan") finally: - P._ado_rest_send, D.broker_manual_cases = o_send, o_cases - assert ok and pid == 9100 and d == "" - # suite create pinned to the 2 configs, no inherited defaults - suite_body = next(b for (u, m, b) in sends if "/suites?" in u) - assert suite_body["name"] == T.BROKER_MANUAL_SUITE_NAME - assert suite_body["inheritDefaultConfigurations"] is False - assert [c["id"] for c in suite_body["defaultConfigurations"]] == T.BROKER_CONFIGS - # each case added with a point assignment per config + (P._ado_rest_send, P._ado_rest_get_all, P._ado_rest_get_h, D.broker_manual_cases) = o + assert ok and pid == 9000 and d == "" + suite_bodies = [b for (u, m, b) in sends if "/suites?" in u] + # Broker flat suite pinned to both flight configs + broker = next(b for b in suite_bodies if b["name"] == T.BROKER_MANUAL_SUITE_NAME) + assert broker["suiteType"] == "staticTestSuite" and broker["inheritDefaultConfigurations"] is False + assert [c["id"] for c in broker["defaultConfigurations"]] == T.BROKER_CONFIGS + # Native Auth folder pinned ECS-only + na = next(b for b in suite_bodies if b["name"] == T.BROKER_NATIVE_AUTH_SUITE_NAME) + assert [c["id"] for c in na["defaultConfigurations"]] == T.NATIVE_AUTH_CONFIGS + # the master's dynamic child recreated WITH its query, ECS-only + dyn = next(b for b in suite_bodies if b["name"] == "Native Auth Test Manual") + assert dyn["suiteType"] == "dynamicTestSuite" and dyn["queryString"].startswith("SELECT") + assert [c["id"] for c in dyn["defaultConfigurations"]] == T.NATIVE_AUTH_CONFIGS + # Broker cases added with both configs add_body = next(b for (u, m, b) in sends if "/TestCase?" in u) - assert len(add_body) == 2 assert [pa["configurationId"] for pa in add_body[0]["pointAssignments"]] == T.BROKER_CONFIGS -def test_create_broker_flat_plan_cleans_up_on_case_failure(): - """If adding cases fails after the plan is created, the half-built plan is DELETEd so a - re-run starts clean, and the function returns (False, ...).""" +def test_build_broker_plan_cleans_up_on_case_failure(): + """If adding the Broker cases fails after the plan is created, the half-built plan is + DELETEd so a re-run starts clean, and the function returns (False, ...).""" from tools import pipelines as P from tools import testplans as T from tools import distribution as D @@ -3204,13 +3229,13 @@ def fake_send(url, method, body, timeout): return (False, None, "HTTP 400: bad point assignment") return (True, {}, "") - o_send, o_cases = P._ado_rest_send, D.broker_manual_cases + o = (P._ado_rest_send, D.broker_manual_cases) P._ado_rest_send = fake_send D.broker_manual_cases = lambda *a, **k: (True, [{"id": "111", "assignee": "a@x"}], "") try: - ok, pid, d = T.create_broker_flat_plan("TEST FLAT plan") + ok, pid, d = T.build_broker_plan("TEST plan") finally: - P._ado_rest_send, D.broker_manual_cases = o_send, o_cases + P._ado_rest_send, D.broker_manual_cases = o assert not ok and "400" in d assert any(m == "DELETE" and "/plans/9200" in u for (m, u) in calls) # cleaned up diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index f7012733..34733b30 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -2,13 +2,14 @@ Two DIFFERENT release procedures, per the team docs: - * BROKER — build a FLAT monthly plan. Instead of ADO's "Copy Test Plan" (which + * BROKER — build a two-folder monthly plan. Instead of ADO's "Copy Test Plan" (which reproduces the master's whole 45-suite tree), create a fresh plan "Android Monthly - Release - " with ONE static suite "Manual Tests (Android Broker)" holding - just the manual-broker cases resolved from the master's subtree (2007357 / suite - 2008656), each pinned to the two flight configs (293 ECS + 330 LocalFlights). Cases are - REFERENCED (shared, not duplicated). Flat = trackable; downstream steps find the suite - by name so it's a drop-in. + Release - " with exactly two top-level folders: (1) "Manual Tests (Android + Broker)" — one FLAT static suite of the manual-broker cases (master 2007357 / subtree + 2008656 resolved), pinned to the two flight configs (293 ECS + 330 LocalFlights); and + (2) "Manual Tests (Native Auth)" — the master's 2864589 subtree copied AS A FOLDER (its + dynamic query suite preserved, kept current), pinned to ECS only (293). Cases are + REFERENCED (shared, not duplicated). Downstream steps find the Broker suite by name. Doc: eng.ms/.../internal-release-checklist/test-plans * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing @@ -43,6 +44,12 @@ # Assigned explicitly so the flat plan gets exactly 2 points/case (matches the master's # matrix) instead of inheriting the project's ~190 default configurations. BROKER_CONFIGS = [293, 330] +# The "Manual Tests (Native Auth)" subtree of the master (static folder + a dynamic query +# suite). Unlike Broker, this is copied AS A FOLDER (structure preserved, not flattened) so +# the monthly plan has exactly two top-level folders. Native Auth runs ECS only (config 293). +BROKER_NATIVE_AUTH_ROOT_SUITE = 2864589 +BROKER_NATIVE_AUTH_SUITE_NAME = "Manual Tests (Native Auth)" +NATIVE_AUTH_CONFIGS = [293] BROKER_AREA_PATH = "Engineering\\Auth Client\\Broker\\Android" BROKER_ITERATION = "Engineering" @@ -142,26 +149,93 @@ def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): # ---------------------------------------------------------------- writes -def create_broker_flat_plan(dest_name, timeout=120): - """Build the release's FLAT Broker test plan and return (ok, new_plan_id, detail). +def _create_suite(pid, parent_id, name, configs, suite_type="staticTestSuite", + query=None, timeout=120): + """Create one suite under `parent_id`, pinned to `configs` (no inherited defaults). + Returns (ok, suite_id, detail).""" + body = {"suiteType": suite_type, "name": name, "parentSuite": {"id": parent_id}, + "inheritDefaultConfigurations": False, + "defaultConfigurations": [{"id": c} for c in configs]} + if query is not None: + body["queryString"] = query + ok, sj, d = P._ado_rest_send( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{pid}/suites?api-version=7.1-preview.1", + "POST", body, timeout) + if not ok: + return (False, None, d) + sid = (sj or {}).get("id") + return (True, sid, "") if sid else (False, None, f"suite '{name}' create returned no id") + + +def _add_cases(pid, suite_id, case_ids, configs, timeout=120): + """Add `case_ids` to `suite_id`, each with a point assignment per config. (ok, detail).""" + if not case_ids: + return (True, "") + body = [{"workItem": {"id": int(cid)}, + "pointAssignments": [{"configurationId": c} for c in configs]} + for cid in case_ids] + ok, _j, d = P._ado_rest_send( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{pid}/Suites/{suite_id}/TestCase" + f"?api-version=7.1-preview.3", "POST", body, max(timeout, 120)) + return (ok, d) + + +def _replicate_subtree(pid, dest_parent, src_children, src_suite_id, configs, timeout=120): + """Recursively copy the master suite `src_suite_id` (and its children) under + `dest_parent` in the new plan, pinned to `configs` — preserving the FOLDER structure + (static folders + dynamic query suites) rather than flattening. `src_children` maps a + master suite id → its child suite dicts. Static suites also get their direct test cases + copied. Returns (ok, detail).""" + for child in src_children.get(src_suite_id, []): + cid, name = child["id"], child.get("name") or f"Suite {child['id']}" + stype = child.get("suiteType") or "staticTestSuite" + query = None + if stype == "dynamicTestSuite": + ok, det, _h, d = P._ado_rest_get_h( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{BROKER_MASTER_PLAN}/suites/{cid}" + f"?api-version=7.1-preview.1", 60) + if not ok: + return (False, f"could not read master suite {cid}: {d}") + query = (det or {}).get("queryString") + okc, new_sid, dc = _create_suite(pid, dest_parent, name, configs, stype, query, timeout) + if not okc: + return (False, dc) + # static suites carry their own directly-added cases → copy them + if stype == "staticTestSuite": + okl, cases, dl = P._ado_rest_get_all( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{BROKER_MASTER_PLAN}/Suites/{cid}" + f"/TestCase?api-version=7.1", timeout) + if not okl: + return (False, dl) + direct = [str(((c.get("workItem") or {}).get("id") + or (c.get("testCase") or {}).get("id"))) + for c in cases if (c.get("workItem") or c.get("testCase"))] + oka, da = _add_cases(pid, new_sid, [i for i in direct if i and i != "None"], + configs, timeout) + if not oka: + return (False, da) + oksub, dsub = _replicate_subtree(pid, new_sid, src_children, cid, configs, timeout) + if not oksub: + return (False, dsub) + return (True, "") + + +def build_broker_plan(dest_name, timeout=120): + """Build the release's Broker test plan and return (ok, new_plan_id, detail). Instead of ADO's "Copy Test Plan" (which reproduces the master's whole 45-suite - hierarchy), this creates a fresh plan with a SINGLE static suite - "Manual Tests (Android Broker)" holding just the relevant manual-broker cases — the - only tests the bug bash runs. Steps: - - 1. create an empty plan `dest_name` (Broker area/iteration); - 2. create one static child suite named "Manual Tests (Android Broker)" pinned to the - two flight configurations (293 ECS + 330 LocalFlights) — inheritDefaultConfigurations - is False so the plan's ~190 project-default configs are NOT applied; - 3. resolve the master's "Manual Tests (Android Broker)" subtree (flattening its dynamic - sub-suites) to the current set of cases, and add them all to the flat suite, - referencing the existing work items (shared, not duplicated), each assigned to both - configs → 2 points/case (matches the master's matrix). - - Downstream (distribute_tests, gather_progress) find the suite by name, so a flat plan is - a drop-in. On a partial failure the half-built plan is best-effort deleted so a re-run - starts clean. + hierarchy), this creates a fresh plan with exactly TWO top-level folders: + + • "Manual Tests (Android Broker)" — a single FLAT static suite of the manual-broker + cases (the master's 2008656 subtree resolved/flattened), pinned to the two flight + configs (293 ECS + 330 LocalFlights) → 2 points/case. + • "Manual Tests (Native Auth)" — copied AS A FOLDER (the master's 2864589 subtree: + a static folder + its dynamic query suite), pinned to ECS only (293). Kept dynamic + so it stays current; not flattened. + + All cases are REFERENCED (shared, not duplicated). Downstream (distribute_tests, + gather_progress) find the Broker suite by name, so this is a drop-in. On any partial + failure the half-built plan is best-effort deleted so a re-run starts clean. """ from tools import distribution as D # local import avoids a circular import @@ -181,33 +255,42 @@ def _cleanup(reason): P._ado_rest_send(f"{ORG}/{PROJECT}/_apis/testplan/plans/{pid}?{_API}", "DELETE", None, 60) return (False, None, reason) - # 2) single flat static suite, pinned to the two flight configs - ok2, sj, d2 = P._ado_rest_send( - f"{ORG}/{PROJECT}/_apis/testplan/Plans/{pid}/suites?api-version=7.1-preview.1", "POST", - {"suiteType": "staticTestSuite", "name": BROKER_MANUAL_SUITE_NAME, - "parentSuite": {"id": root}, "inheritDefaultConfigurations": False, - "defaultConfigurations": [{"id": c} for c in BROKER_CONFIGS]}, - timeout) - if not ok2: - return _cleanup(f"suite create failed: {d2}") - suite_id = (sj or {}).get("id") - if not suite_id: - return _cleanup("suite create returned no id") - - # 3) resolve the master's manual-broker cases and add them to the flat suite + # 2) FLAT Broker suite (all manual-broker cases resolved from the master subtree) + okb, broker_suite, db = _create_suite(pid, root, BROKER_MANUAL_SUITE_NAME, + BROKER_CONFIGS, timeout=timeout) + if not okb: + return _cleanup(f"Broker suite create failed: {db}") okc, cases, dc = D.broker_manual_cases(BROKER_MASTER_PLAN, BROKER_MANUAL_ROOT_SUITE, timeout) if not okc: - return _cleanup(f"could not resolve master manual cases: {dc}") + return _cleanup(f"could not resolve master manual-broker cases: {dc}") if not cases: return _cleanup("master 'Manual Tests (Android Broker)' subtree resolved to 0 cases") - body = [{"workItem": {"id": int(c["id"])}, - "pointAssignments": [{"configurationId": cid} for cid in BROKER_CONFIGS]} - for c in cases] - oka, _aj, da = P._ado_rest_send( - f"{ORG}/{PROJECT}/_apis/testplan/Plans/{pid}/Suites/{suite_id}/TestCase" - f"?api-version=7.1-preview.3", "POST", body, max(timeout, 120)) + oka, da = _add_cases(pid, broker_suite, [c["id"] for c in cases], BROKER_CONFIGS, timeout) if not oka: - return _cleanup(f"adding {len(cases)} cases failed: {da}") + return _cleanup(f"adding {len(cases)} Broker cases failed: {da}") + + # 3) Native Auth FOLDER (structure preserved) — replicate the master's subtree + okm, masters, dm = P._ado_rest_get_all( + f"{ORG}/{PROJECT}/_apis/testplan/Plans/{BROKER_MASTER_PLAN}/suites?{_API}", timeout) + if not okm: + return _cleanup(f"could not read master suites: {dm}") + src_children = {} + na_src = None + for s in masters: + src_children.setdefault((s.get("parentSuite") or {}).get("id"), []).append(s) + if s.get("id") == BROKER_NATIVE_AUTH_ROOT_SUITE: + na_src = s + if na_src is None: + return _cleanup(f"master Native Auth suite #{BROKER_NATIVE_AUTH_ROOT_SUITE} not found") + okna, na_folder, dna = _create_suite(pid, root, BROKER_NATIVE_AUTH_SUITE_NAME, + NATIVE_AUTH_CONFIGS, timeout=timeout) + if not okna: + return _cleanup(f"Native Auth folder create failed: {dna}") + okr, dr = _replicate_subtree(pid, na_folder, src_children, BROKER_NATIVE_AUTH_ROOT_SUITE, + NATIVE_AUTH_CONFIGS, timeout) + if not okr: + return _cleanup(f"replicating Native Auth folder failed: {dr}") + return (True, pid, "") From 54a9ec22efbb31cb6010bf60047357116214e390 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Mon, 24 Aug 2026 20:16:05 +0100 Subject: [PATCH 89/90] Phase 3 bug bash: notify Native Auth RE + gather sign-offs Add two scout steps. notify_native_auth: one-shot Teams ping to the month's Native Auth release engineer (from the RE schedule) that the bug bash is ready, asking them to run their Native Auth tests and confirm to the owner when done. signoffs: RE attests the Native Auth team signed off + Scout requests DID sign-off from the DID contact (Sowmya Malayanur); completes when both are recorded. Reorder Phase 3 so scout steps precede human touchpoints. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 53 +++++++ release-agent/config/phases.yaml | 3 +- .../orchestrator/commands/bugbash_chat.py | 88 +++++++++++ .../steps/bug_bash/notify_native_auth.py | 121 ++++++++++++++++ release-agent/steps/bug_bash/signoffs.py | 91 ++++++++++++ release-agent/tests/test_engine.py | 137 +++++++++++++++++- 6 files changed, 490 insertions(+), 3 deletions(-) create mode 100644 release-agent/steps/bug_bash/notify_native_auth.py create mode 100644 release-agent/steps/bug_bash/signoffs.py diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 678cb387..a14921d9 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -460,6 +460,34 @@ bug_bash.send_invite: - q: "Where does the template live?" a: "templates/bug-bash-invite.html — edit it to restyle; the step always renders from that file." +bug_bash.notify_native_auth: + summary: "One-shot Teams ping to the Native Auth release engineer that the bug bash is ready." + what: > + Sends ONE Teams message to the month's Native Auth Release Engineer (from the + release-engineer schedule doc) telling them the bug bash is ready, that the Broker test + plan now has a 'Manual Tests (Native Auth)' folder with their cases (linked), asking them + to run their Native Auth tests at their earliest convenience, and to send the release + owner a confirmation when done. Fired once and done — no poller/automation. + who: > + Scout runs it during `next` (a scout step). It resolves the Native Auth RE for the release + month from the schedule doc (an inline hint seeds the likely alias; the skill confirms + against the live doc), resolves their UPN (workiq_search_people), and sends a 1:1 Teams + message (workiq_send_chat_message). Human fallback: if it can't resolve them it asks the + owner for the alias, or holds the step. + how: > + The skill runs `record-nativeauth-notify --release --engineer ` to mark + it done (stores who was notified); without --engineer it holds the step for the owner. + Idempotent — once an engineer is recorded it reports done without re-sending. Blocks if the + Broker plan isn't built yet (needs the Native Auth suite link). + links: + - name: "Release engineer schedule (Native Auth RE column)" + url: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/internal-release-checklist/release-engineer-schedule" + faqs: + - q: "Who receives the message?" + a: "The Native Auth Release Engineer for the release month, from the schedule doc (e.g. Aug 2026 → silviu.petrescu). The inline hint in the step is confirmed against the live doc; update the hint when the schedule changes." + - q: "Does it track their confirmation?" + a: "No — it just asks them to reply to the release owner when they finish. It's a one-shot notification, not a poller." + bug_bash.activate_chat: summary: "Activate the Bug Bash meeting chat and store its id for the periodic updates." what: > @@ -518,6 +546,31 @@ bug_bash.bugbash_updates: - q: "Why group by AssignedTo instead of the test point's tester?" a: "The ADO test-point `tester` field is NOT synced with the case's System.AssignedTo. distribute_tests sets AssignedTo, so progress is grouped by AssignedTo to reflect who actually owns each test." +bug_bash.signoffs: + summary: "Gather the two release sign-offs — Native Auth (attest) + DID (from Sowmya)." + what: > + Two sign-offs close the bug bash before the owner's final attestation: (1) NATIVE AUTH — + the release engineer attests the Native Auth team finished their tests and signed off (the + RE received their confirmation, per notify_native_auth); (2) DID — reach out to the DID + point of contact (Sowmya Malayanur, Sowmya.Malayanur@microsoft.com) for the DID sign-off. + The step completes only when BOTH are in. + who: > + Scout runs it during `next` (a scout step). It composes the DID sign-off request and sends + it to the DID contact (workiq_create_chat_by_email → workiq_send_chat_message); the release + engineer confirms the Native Auth sign-off. Until both are recorded the step holds. + how: > + Record with `record-signoffs --release --native-auth-by '' --did-by ''` + (both → done; either missing → holds for the owner). Idempotent — once both are recorded it + reports done without re-sending the DID request. + links: + - name: "DID point of contact — Sowmya Malayanur" + url: "mailto:Sowmya.Malayanur@microsoft.com" + faqs: + - q: "Who is the DID contact?" + a: "Sowmya Malayanur (Sowmya.Malayanur@microsoft.com) — Scout sends her the DID sign-off request when this step runs." + - q: "What does the release engineer attest?" + a: "That the Native Auth team finished their tests and signed off — the confirmation they were asked to send back in notify_native_auth." + bug_bash.bugbash_complete: summary: "The release owner attests the Bug Bash is finished — the gate that closes the bash." what: > diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 864c50e7..fe69af4f 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -67,10 +67,11 @@ phases: - { id: clone_plans_auth, name: "Create Authenticator bug-bash test suite", owner: agent, maps_to: [T3] } - { id: distribute_tests, name: "Distribute bug-bash tests across the team", owner: agent, maps_to: [T2] } - { id: send_invite, name: "Schedule Bug Bash + send Teams invite", owner: agent, source: scout, maps_to: [T2] } + - { id: notify_native_auth, name: "Notify the Native Auth release engineer", owner: agent, source: scout, maps_to: [T2] } - { id: activate_chat, name: "Activate the Bug Bash meeting chat + store its id", owner: agent, source: scout, maps_to: [T2] } - { id: bugbash_updates, name: "Post bug-bash progress updates (first post + 2h poller)", owner: agent, source: scout, maps_to: [T2] } + - { id: signoffs, name: "Native Auth sign-off + DID sign-off", owner: agent, source: scout, maps_to: [T5] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", owner: agent, maps_to: [T5] } - { id: bugbash_complete, name: "Bug bash complete + signed off", owner: human, source: scout, gate: true } - id: finalize diff --git a/release-agent/orchestrator/commands/bugbash_chat.py b/release-agent/orchestrator/commands/bugbash_chat.py index 5e1a002a..dad10ac4 100644 --- a/release-agent/orchestrator/commands/bugbash_chat.py +++ b/release-agent/orchestrator/commands/bugbash_chat.py @@ -48,6 +48,76 @@ def cmd_record_bugbash_chat(args): return 0 +def cmd_record_nativeauth_notify(args): + """Record that the Native Auth release engineer was notified the bug bash is ready. + * --engineer given -> store it (data.engineer) + mark notify_native_auth done. + * --engineer omitted -> hold the step for the owner (attention): couldn't resolve/send. + """ + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + eng = (args.engineer or "").strip() + + if not eng: + detail = ("Could not resolve/notify the Native Auth release engineer. Check the " + "release-engineer schedule and send them the bug-bash-ready message, then " + "re-run this step with --engineer.") + orch.record_scout_step("bug_bash", "notify_native_auth", "attention", detail) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f"[attention] notify_native_auth: {detail}", kind="step") + print(detail) + return 2 + + orch.record_scout_step("bug_bash", "notify_native_auth", "pass", + f"Native Auth RE notified: {eng}") + step = orch.state.get_step("bug_bash", "notify_native_auth") + step.data = dict(step.data or {}) + step.data["engineer"] = eng + step.by = "scout" + orch.state.set_step("bug_bash", "notify_native_auth", step) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f"[ok] notify_native_auth: Native Auth RE notified ({eng})", kind="step") + print(f"Recorded Native Auth RE notification: {eng}") + return 0 + + +def cmd_record_signoffs(args): + """Record the two bug-bash sign-offs (Native Auth attest + DID from Sowmya). + * both --native-auth-by and --did-by given -> store + mark signoffs done. + * either missing -> hold the step for the owner (attention): sign-offs still pending. + """ + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + na = (args.native_auth_by or "").strip() + did = (args.did_by or "").strip() + + if not (na and did): + pending = ", ".join(p for p, v in + [("Native Auth", na), ("DID", did)] if not v) + detail = (f"Sign-offs still pending: {pending}. Native Auth = the release engineer " + f"attests the Native Auth team finished + signed off; DID = sign-off from " + f"the DID contact (Sowmya Malayanur). Re-run with both --native-auth-by and " + f"--did-by once received.") + orch.record_scout_step("bug_bash", "signoffs", "attention", detail) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, f"[attention] signoffs: {detail}", kind="step") + print(detail) + return 2 + + orch.record_scout_step("bug_bash", "signoffs", "pass", + f"Sign-offs complete: Native Auth ({na}) + DID ({did})") + step = orch.state.get_step("bug_bash", "signoffs") + step.data = dict(step.data or {}) + step.data["native_auth_by"] = na + step.data["did_by"] = did + step.by = "scout" + orch.state.set_step("bug_bash", "signoffs", step) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f"[ok] signoffs: Native Auth ({na}) + DID ({did}) signed off", kind="step") + print(f"Recorded sign-offs: Native Auth={na}, DID={did}") + return 0 + + def register(sub): p = sub.add_parser("record-bugbash-chat", help="Store the resolved Bug Bash meeting chat id (Phase-3 activate_chat)") @@ -56,3 +126,21 @@ def register(sub): help="The meeting chat id (19:meeting_…@thread.v2). Omit to hold for the owner.") p.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") p.set_defaults(func=cmd_record_bugbash_chat) + + n = sub.add_parser("record-nativeauth-notify", + help="Record that the Native Auth release engineer was notified the bug bash is ready") + n.add_argument("--release", required=True) + n.add_argument("--engineer", default=None, + help="Alias or UPN of the notified Native Auth RE. Omit to hold for the owner.") + n.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + n.set_defaults(func=cmd_record_nativeauth_notify) + + s = sub.add_parser("record-signoffs", + help="Record the bug-bash sign-offs (Native Auth attest + DID from Sowmya)") + s.add_argument("--release", required=True) + s.add_argument("--native-auth-by", default=None, dest="native_auth_by", + help="Who signed off Native Auth. Omit (with --did-by) to hold the step.") + s.add_argument("--did-by", default=None, dest="did_by", + help="Who gave the DID sign-off. Omit to hold the step.") + s.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + s.set_defaults(func=cmd_record_signoffs) diff --git a/release-agent/steps/bug_bash/notify_native_auth.py b/release-agent/steps/bug_bash/notify_native_auth.py new file mode 100644 index 00000000..e9f693ac --- /dev/null +++ b/release-agent/steps/bug_bash/notify_native_auth.py @@ -0,0 +1,121 @@ +"""Step: `notify_native_auth` — one-shot Teams ping to the Native Auth release engineer that +the bug bash is ready (Phase 3, bug_bash). + +The Broker test plan now carries a "Manual Tests (Native Auth)" folder (see +clone_plans_broker). This step sends ONE Teams message to the month's Native Auth Release +Engineer — from the release-engineer schedule — telling them the bug bash is ready, asking +them to run their Native Auth tests at their earliest convenience, and to send the release +owner a confirmation when they finish. It is fired once and done — no poller, no automation. + +Scout-assisted (skill-driven), three tiers: + 1. RESOLVE — read the Native Auth Release Engineer for from the schedule + doc (link below). The inline NATIVE_AUTH_RE hint seeds the likely alias; the skill + confirms/overrides it from the live doc. + 2. SEND — resolve that engineer's UPN (workiq_search_people) and send them the 1:1 Teams + message (workiq_send_chat_message / SendMessageToUser) with the text the engine composed. + 3. HUMAN FALLBACK — if the engineer can't be resolved, ask the owner who the Native Auth RE + is (m_ask_user), or hold the step. + +The skill records completion with `record-nativeauth-notify --release --engineer ` +(or without --engineer to hold the step for the owner). + +Depends on clone_plans_broker (for the Native Auth suite link). Idempotent — once recorded +with an engineer it reports done without re-sending. +""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator.outcomes import NeedsSkill, Blocked, Done +from tools import testplans as T + +ID = "notify_native_auth" +KIND = "scout" + +SCHEDULE_DOC = ("https://eng.ms/docs/microsoft-security/identity/entra-developer-application-" + "platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/" + "internal-release-checklist/release-engineer-schedule") + +# Native Auth Release Engineer per release month, seeded from the schedule doc. A HINT only — +# the skill confirms against the live doc and may override. Keyed 'Month YYYY'. +NATIVE_AUTH_RE = { + "March 2026": "mmizrak", "April 2026": "mmizrak", "May 2026": "mmizrak", + "June 2026": "mmizrak", "July 2026": "mmizrak", + "August 2026": "silviu.petrescu", "September 2026": "mmizrak", + "October 2026": "silviu.petrescu", "November 2026": "mmizrak", + "January 2027": "silviu.petrescu", "February 2027": "mmizrak", "March 2027": "silviu.petrescu", +} + + +def notified_engineer(state): + """The alias/UPN the step recorded as notified, or None.""" + return (state.get_step("bug_bash", ID).data or {}).get("engineer") + + +def _message_html(month_year, owner_name, plan_url): + who = owner_name or "the release owner" + return ( + f'
    ' + f'

    Hi \u2014 the {month_year} Android release Bug Bash is ready to go.

    ' + f'

    This month the Broker test plan includes a ' + f'Manual Tests (Native Auth) folder with your Native ' + f'Auth test cases. Could you please run your Native Auth tests at your earliest ' + f'convenience and mark pass/fail in the test plan?

    ' + f'

    When you\u2019re done, please send {who} a quick confirmation message so ' + f'we can close out the bash. Thanks!

    ' + f'
    ') + + +def build(state): + if not state.ccd: + return Blocked("notify_native_auth: no CCD set — can't identify the release month.") + broker_plan = (state.get_step("bug_bash", "clone_plans_broker").data or {}).get("plan_id") + if not broker_plan: + return Blocked("notify_native_auth: the Broker test plan isn't built yet " + "(clone_plans_broker) — run that first.") + + # Idempotent: already recorded as notified → done, no re-send. + already = notified_engineer(state) + if already: + return Done(f"Native Auth RE ({already}) already notified the {ID} for " + f"{state.release_id}.") + + month_year = schedule.parse_date(state.ccd).strftime("%B %Y") + re_hint = NATIVE_AUTH_RE.get(month_year, "") + plan_url = T.plan_web_url(broker_plan) + html = _message_html(month_year, state.owner_name, plan_url) + + hint_txt = (f"The schedule lists '{re_hint}' as the Native Auth RE for {month_year} " + f"(confirm against the doc)." if re_hint + else f"The inline hint has no entry for {month_year} — read it from the doc.") + instructions = ( + f"Notify the Native Auth Release Engineer that the {month_year} bug bash is ready.\n" + f"1. RESOLVE the Native Auth RE for '{month_year}' from the schedule doc " + f"({SCHEDULE_DOC}) — the 'Native Auth Release Engineer' column (aliases are in " + f"parens, e.g. 'Silviu (silviu.petrescu)'). {hint_txt}\n" + f"2. SEND the message below to that engineer as a 1:1 Teams message: resolve their " + f"UPN with `workiq_search_people` (or the alias@microsoft.com), then " + f"`workiq_send_chat_message` (or SendMessageToUser). contentType html.\n" + f"3. HUMAN FALLBACK if you can't resolve them: `m_ask_user` for the Native Auth RE's " + f"alias/email, then send.\n" + f"4. Record: `record-nativeauth-notify --release {state.release_id} --engineer " + f"''`. If it truly can't be sent, run WITHOUT --engineer to hold it.") + + return NeedsSkill( + tool="record-nativeauth-notify", + payload={ + "release": state.release_id, + "engineer_hint": re_hint, + "content": html, + "contentType": "html", + "followup_command": (f"record-nativeauth-notify --release {state.release_id} " + f"--engineer ''"), + "_gather": {"month_year": month_year, "schedule_doc": SCHEDULE_DOC, + "instructions": instructions}, + }, + record_as=ID, + summary=(f"Notify the {month_year} Native Auth RE" + + (f" ({re_hint})" if re_hint else "") + + " that the bug bash is ready + ask for a confirmation when done"), + note="awaiting Native Auth RE notification (resolve from schedule → send → record)", + outbound=True, + ) diff --git a/release-agent/steps/bug_bash/signoffs.py b/release-agent/steps/bug_bash/signoffs.py new file mode 100644 index 00000000..2e4fbb73 --- /dev/null +++ b/release-agent/steps/bug_bash/signoffs.py @@ -0,0 +1,91 @@ +"""Step: `signoffs` — gather the two release sign-offs (Phase 3, bug_bash). + +Two sign-offs close the bug bash before the owner's final attestation (bugbash_complete): + + 1. NATIVE AUTH — the release engineer attests the Native Auth team finished their tests and + signed off (the RE received their confirmation, per notify_native_auth). + 2. DID — reach out to the DID point of contact (Sowmya Malayanur) for the DID sign-off. + +Scout-assisted: build() composes the DID sign-off request and instructs the skill to send it +to the DID contact, then record both sign-offs. The step completes only when BOTH are +confirmed; otherwise it holds (the DID request having been sent). + +The skill records with `record-signoffs --release --native-auth-by '' --did-by +''` (both → done; missing either → hold for the owner). Idempotent — once both are +recorded it reports done without re-sending. +""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator.outcomes import NeedsSkill, Blocked, Done +from tools import testplans as T + +ID = "signoffs" +KIND = "scout" + +# DID (Directory / Identity) sign-off point of contact. +DID_CONTACT = {"name": "Sowmya Malayanur", "email": "Sowmya.Malayanur@microsoft.com"} + + +def recorded_signoffs(state): + """(native_auth_by, did_by) recorded on the step, each or None.""" + d = state.get_step("bug_bash", ID).data or {} + return (d.get("native_auth_by"), d.get("did_by")) + + +def _did_request_html(month_year, plan_url): + plan_line = (f' Test plan: {month_year} Broker test plan.' + if plan_url else "") + return ( + f'
    ' + f'

    Hi {DID_CONTACT["name"].split()[0]} \u2014 the {month_year} Android release ' + f'Bug Bash testing is complete on our side (Broker + Native Auth).

    ' + f'

    Could you please provide the DID sign-off when you get a chance?{plan_line}' + f' Thanks!

    ') + + +def build(state): + if not state.ccd: + return Blocked("signoffs: no CCD set — can't identify the release month.") + + na_by, did_by = recorded_signoffs(state) + if na_by and did_by: + return Done(f"Sign-offs complete for {state.release_id}: Native Auth ({na_by}) + " + f"DID ({did_by}).") + + month_year = schedule.parse_date(state.ccd).strftime("%B %Y") + broker_plan = (state.get_step("bug_bash", "clone_plans_broker").data or {}).get("plan_id") + plan_url = T.plan_web_url(broker_plan) if broker_plan else "" + html = _did_request_html(month_year, plan_url) + + instructions = ( + f"Gather the two {month_year} bug-bash sign-offs, then record them.\n" + f"1. NATIVE AUTH: confirm with the release engineer that the Native Auth team " + f"finished their tests and signed off (the RE should have received the confirmation " + f"requested in notify_native_auth). Capture who signed off.\n" + f"2. DID: send the message below to the DID point of contact " + f"({DID_CONTACT['name']}, {DID_CONTACT['email']}) as a 1:1 Teams message " + f"(`workiq_create_chat_by_email` \u2192 `workiq_send_chat_message`, contentType html), " + f"requesting the DID sign-off. Capture their sign-off when it comes back.\n" + f"3. Record when BOTH are in: `record-signoffs --release {state.release_id} " + f"--native-auth-by '' --did-by ''`. If either is still pending, run " + f"`record-signoffs --release {state.release_id}` (no flags) to hold the step.") + + return NeedsSkill( + tool="record-signoffs", + payload={ + "release": state.release_id, + "content": html, + "contentType": "html", + "did_contact": DID_CONTACT, + "followup_command": (f"record-signoffs --release {state.release_id} " + f"--native-auth-by '' --did-by ''"), + "_gather": {"month_year": month_year, "did_contact": DID_CONTACT, + "instructions": instructions}, + }, + record_as=ID, + summary=(f"Attest Native Auth sign-off + request DID sign-off from " + f"{DID_CONTACT['name']} for the {month_year} bug bash"), + note="awaiting sign-offs (Native Auth attest + DID from Sowmya Malayanur)", + outbound=True, + ) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 6788c454..29263dc2 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -77,8 +77,10 @@ def _recent_iso(): "bug_bash.clone_plans_auth": {"outcome": "done", "note": "auth suite created (test)"}, "bug_bash.distribute_tests": {"outcome": "done", "note": "tests distributed (test)"}, "bug_bash.send_invite": {"outcome": "done", "note": "bug bash invite sent (test)"}, + "bug_bash.notify_native_auth": {"outcome": "done", "note": "native auth RE notified (test)"}, "bug_bash.activate_chat": {"outcome": "done", "note": "meeting chat activated (test)"}, "bug_bash.bugbash_updates": {"outcome": "done", "note": "first bug-bash update posted (test)"}, + "bug_bash.signoffs": {"outcome": "done", "note": "sign-offs recorded (test)"}, } @@ -644,8 +646,8 @@ def test_holds_at_first_hold(): # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/ # distribute_tests (3) + send_invite + activate_chat (scout, mocked done) (2) + - # bugbash_updates (scout, mocked done) (1). - assert sum(1 for a in actions if a.kind == "ran") == 15 + # notify_native_auth (1) + bugbash_updates (1) + signoffs (scout, mocked done) (1). + assert sum(1 for a in actions if a.kind == "ran") == 17 def test_gate_blocks_until_approved(): @@ -2995,6 +2997,137 @@ def test_record_bugbash_chat_stores_id_and_marks_done(): assert after.get_step("bug_bash", "activate_chat").status == "blocked" +# ---- Phase 3: notify_native_auth ---- + +def _na_state(release="2026-08", ccd="2026-08-13", broker_plan=3730001, owner="Pedro"): + from orchestrator.state import StepState + st = ReleaseState(release_id=release, ccd=ccd, owner_name=owner) + st.set_step("bug_bash", "clone_plans_broker", StepState(status="done", + data={"plan_id": broker_plan})) + return st + + +def test_notify_native_auth_composes_needs_skill(): + """notify_native_auth is a scout step: NeedsSkill with the RE-resolution instructions, + the composed message (linking the Native Auth suite + asking for a confirmation), and the + recorder follow-up. Seeds the Aug 2026 RE hint from the schedule.""" + import steps as _steps + from orchestrator.outcomes import as_dict + st = _na_state() + out = as_dict(_steps.get_step("bug_bash", "notify_native_auth").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "record-nativeauth-notify" + assert out["payload"]["engineer_hint"] == "silviu.petrescu" # Aug 2026 from schedule + assert "planId=3730001" in out["payload"]["content"] # links the Broker plan + assert "confirmation" in out["payload"]["content"].lower() + assert "record-nativeauth-notify --release 2026-08" in out["payload"]["followup_command"] + assert "release-engineer-schedule" in out["payload"]["_gather"]["schedule_doc"] + assert _steps.get_step("bug_bash", "notify_native_auth").KIND == "scout" + + +def test_notify_native_auth_blocks_without_broker_plan(): + import steps as _steps + from orchestrator.outcomes import as_dict + st = ReleaseState(release_id="2026-08", ccd="2026-08-13") + out = as_dict(_steps.get_step("bug_bash", "notify_native_auth").build(st)) + assert out["kind"] == "blocked" and "Broker test plan" in out["reason"] + + +def test_notify_native_auth_idempotent_after_recorded(): + import steps as _steps + from orchestrator.outcomes import as_dict + from orchestrator.state import StepState + st = _na_state() + st.set_step("bug_bash", "notify_native_auth", + StepState(status="done", data={"engineer": "silviu.petrescu"})) + out = as_dict(_steps.get_step("bug_bash", "notify_native_auth").build(st)) + assert out["kind"] == "done" and "already notified" in out["note"] + + +def test_record_nativeauth_notify_stores_or_holds(): + """record-nativeauth-notify with --engineer stores it + marks done; without --engineer it + holds the step for the owner.""" + import tempfile, argparse + from orchestrator import cli_common as _C + from orchestrator.commands import bugbash_chat as BC + from steps.bug_bash.notify_native_auth import notified_engineer + with tempfile.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + _C.save_state(_na_state(), d, rid) + ns = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, + engineer="silviu.petrescu") + assert BC.cmd_record_nativeauth_notify(ns) == 0 + again = _C.load_state(d, rid) + assert again.is_done("bug_bash", "notify_native_auth") + assert notified_engineer(again) == "silviu.petrescu" + + # no engineer -> attention hold + _C.save_state(_na_state(), d, rid) + ns2 = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, engineer=None) + assert BC.cmd_record_nativeauth_notify(ns2) == 2 + after = _C.load_state(d, rid) + assert not after.is_done("bug_bash", "notify_native_auth") + assert after.get_step("bug_bash", "notify_native_auth").status == "blocked" + + +# ---- Phase 3: signoffs (Native Auth attest + DID) ---- + +def test_signoffs_composes_needs_skill_with_did_request(): + """signoffs is a scout step: NeedsSkill composing the DID sign-off request to Sowmya + + instructions to gather the Native Auth attest, with the recorder follow-up.""" + import steps as _steps + from orchestrator.outcomes import as_dict + from steps.bug_bash.signoffs import DID_CONTACT + st = _na_state() + out = as_dict(_steps.get_step("bug_bash", "signoffs").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "record-signoffs" + assert out["payload"]["did_contact"]["email"] == "Sowmya.Malayanur@microsoft.com" + assert DID_CONTACT["name"].split()[0] in out["payload"]["content"] # greets Sowmya + assert "DID sign-off" in out["payload"]["content"] + assert "planId=3730001" in out["payload"]["content"] # links the plan + assert "record-signoffs --release 2026-08" in out["payload"]["followup_command"] + assert out["outbound"] is True + assert _steps.get_step("bug_bash", "signoffs").KIND == "scout" + + +def test_signoffs_idempotent_when_both_recorded(): + import steps as _steps + from orchestrator.outcomes import as_dict + from orchestrator.state import StepState + st = _na_state() + st.set_step("bug_bash", "signoffs", + StepState(status="done", data={"native_auth_by": "silviu", "did_by": "sowmya"})) + out = as_dict(_steps.get_step("bug_bash", "signoffs").build(st)) + assert out["kind"] == "done" and "Sign-offs complete" in out["note"] + + +def test_record_signoffs_requires_both(): + """record-signoffs marks done only when BOTH sign-offs are given; missing either holds.""" + import tempfile, argparse + from orchestrator import cli_common as _C + from orchestrator.commands import bugbash_chat as BC + from steps.bug_bash.signoffs import recorded_signoffs + with tempfile.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + # only one → hold + _C.save_state(_na_state(), d, rid) + ns = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, + native_auth_by="silviu", did_by=None) + assert BC.cmd_record_signoffs(ns) == 2 + held = _C.load_state(d, rid) + assert not held.is_done("bug_bash", "signoffs") + assert held.get_step("bug_bash", "signoffs").status == "blocked" + + # both → done + ns2 = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, + native_auth_by="silviu", did_by="sowmya") + assert BC.cmd_record_signoffs(ns2) == 0 + done = _C.load_state(d, rid) + assert done.is_done("bug_bash", "signoffs") + assert recorded_signoffs(done) == ("silviu", "sowmya") + + # ---- Phase 3: bugbash_updates (periodic poster) ---- def test_bugbash_holidays_and_working_window(): From a4d09427c25759a5162095e980af166dd728db5b Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Mon, 24 Aug 2026 20:40:52 +0100 Subject: [PATCH 90/90] Phase 3 bug bash: split signoffs into two steps Separate the combined signoffs step into native_auth_signoff (the release engineer attests the Native Auth team signed off; no message) and did_signoff (Scout sends the DID sign-off request to the DID contact and records the reply). Each holds and completes independently. Replaces record-signoffs with record-native-auth- signoff and record-did-signoff. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 44 +++++---- release-agent/config/phases.yaml | 3 +- .../orchestrator/commands/bugbash_chat.py | 70 +++++++------- release-agent/steps/bug_bash/did_signoff.py | 82 +++++++++++++++++ .../steps/bug_bash/native_auth_signoff.py | 56 ++++++++++++ release-agent/steps/bug_bash/signoffs.py | 91 ------------------- release-agent/tests/test_engine.py | 91 ++++++++++--------- 7 files changed, 255 insertions(+), 182 deletions(-) create mode 100644 release-agent/steps/bug_bash/did_signoff.py create mode 100644 release-agent/steps/bug_bash/native_auth_signoff.py delete mode 100644 release-agent/steps/bug_bash/signoffs.py diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index a14921d9..40f4cdf6 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -546,30 +546,42 @@ bug_bash.bugbash_updates: - q: "Why group by AssignedTo instead of the test point's tester?" a: "The ADO test-point `tester` field is NOT synced with the case's System.AssignedTo. distribute_tests sets AssignedTo, so progress is grouped by AssignedTo to reflect who actually owns each test." -bug_bash.signoffs: - summary: "Gather the two release sign-offs — Native Auth (attest) + DID (from Sowmya)." - what: > - Two sign-offs close the bug bash before the owner's final attestation: (1) NATIVE AUTH — - the release engineer attests the Native Auth team finished their tests and signed off (the - RE received their confirmation, per notify_native_auth); (2) DID — reach out to the DID - point of contact (Sowmya Malayanur, Sowmya.Malayanur@microsoft.com) for the DID sign-off. - The step completes only when BOTH are in. +bug_bash.native_auth_signoff: + summary: "The release engineer attests the Native Auth team finished + signed off." + what: > + The first of the two bug-bash sign-offs. No message is sent — the release engineer + confirms the Native Auth team completed their tests and signed off (the confirmation they + were asked to send back in notify_native_auth). The step completes once who signed off is + recorded. + who: "Scout runs it during `next` (a scout step); the release engineer confirms the sign-off." + how: > + Record with `record-native-auth-signoff --release --by ''` (marks done); without + --by it holds for the owner until the sign-off arrives. Idempotent. + faqs: + - q: "What is this confirming?" + a: "That the Native Auth team ran their tests and signed off — the confirmation they were asked to send back in notify_native_auth." + +bug_bash.did_signoff: + summary: "Request the DID sign-off from the DID point of contact (Sowmya Malayanur)." + what: > + The second of the two bug-bash sign-offs. Scout sends a 1:1 Teams message to the DID point + of contact (Sowmya Malayanur, Sowmya.Malayanur@microsoft.com) asking for the DID sign-off + (the message links the Broker test plan), then records it when it comes back. Completes + once the DID sign-off is captured. who: > - Scout runs it during `next` (a scout step). It composes the DID sign-off request and sends - it to the DID contact (workiq_create_chat_by_email → workiq_send_chat_message); the release - engineer confirms the Native Auth sign-off. Until both are recorded the step holds. + Scout runs it during `next` (a scout step). It composes the DID request and sends it to + the DID contact (workiq_create_chat_by_email → workiq_send_chat_message); until the + sign-off is recorded the step holds. how: > - Record with `record-signoffs --release --native-auth-by '' --did-by ''` - (both → done; either missing → holds for the owner). Idempotent — once both are recorded it - reports done without re-sending the DID request. + Record with `record-did-signoff --release --by ''` (marks done); without --by it + holds for the owner (the request having been sent). Idempotent — once recorded it reports + done without re-sending. links: - name: "DID point of contact — Sowmya Malayanur" url: "mailto:Sowmya.Malayanur@microsoft.com" faqs: - q: "Who is the DID contact?" a: "Sowmya Malayanur (Sowmya.Malayanur@microsoft.com) — Scout sends her the DID sign-off request when this step runs." - - q: "What does the release engineer attest?" - a: "That the Native Auth team finished their tests and signed off — the confirmation they were asked to send back in notify_native_auth." bug_bash.bugbash_complete: summary: "The release owner attests the Bug Bash is finished — the gate that closes the bash." diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index fe69af4f..9f1ede4e 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -70,7 +70,8 @@ phases: - { id: notify_native_auth, name: "Notify the Native Auth release engineer", owner: agent, source: scout, maps_to: [T2] } - { id: activate_chat, name: "Activate the Bug Bash meeting chat + store its id", owner: agent, source: scout, maps_to: [T2] } - { id: bugbash_updates, name: "Post bug-bash progress updates (first post + 2h poller)", owner: agent, source: scout, maps_to: [T2] } - - { id: signoffs, name: "Native Auth sign-off + DID sign-off", owner: agent, source: scout, maps_to: [T5] } + - { id: native_auth_signoff, name: "Native Auth team sign-off (attest)", owner: agent, source: scout, maps_to: [T5] } + - { id: did_signoff, name: "DID sign-off (request from DID contact)", owner: agent, source: scout, maps_to: [T5] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - { id: bugbash_complete, name: "Bug bash complete + signed off", owner: human, source: scout, gate: true } diff --git a/release-agent/orchestrator/commands/bugbash_chat.py b/release-agent/orchestrator/commands/bugbash_chat.py index dad10ac4..39e1e00a 100644 --- a/release-agent/orchestrator/commands/bugbash_chat.py +++ b/release-agent/orchestrator/commands/bugbash_chat.py @@ -81,40 +81,40 @@ def cmd_record_nativeauth_notify(args): return 0 -def cmd_record_signoffs(args): - """Record the two bug-bash sign-offs (Native Auth attest + DID from Sowmya). - * both --native-auth-by and --did-by given -> store + mark signoffs done. - * either missing -> hold the step for the owner (attention): sign-offs still pending. - """ +def cmd_record_native_auth_signoff(args): + """Record the Native Auth sign-off (release engineer attests the Native Auth team signed + off). --by given → store + mark done; omitted → hold the step for the owner.""" + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + return _record_signoff(orch, args, "native_auth_signoff", "Native Auth") + + +def cmd_record_did_signoff(args): + """Record the DID sign-off (from the DID contact, Sowmya Malayanur). --by given → store + + mark done; omitted → hold the step for the owner.""" _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) - na = (args.native_auth_by or "").strip() - did = (args.did_by or "").strip() - - if not (na and did): - pending = ", ".join(p for p, v in - [("Native Auth", na), ("DID", did)] if not v) - detail = (f"Sign-offs still pending: {pending}. Native Auth = the release engineer " - f"attests the Native Auth team finished + signed off; DID = sign-off from " - f"the DID contact (Sowmya Malayanur). Re-run with both --native-auth-by and " - f"--did-by once received.") - orch.record_scout_step("bug_bash", "signoffs", "attention", detail) + return _record_signoff(orch, args, "did_signoff", "DID") + + +def _record_signoff(orch, args, step_id, label): + by = (args.by or "").strip() + if not by: + detail = (f"{label} sign-off still pending. Re-run with --by '' once it's " + f"received.") + orch.record_scout_step("bug_bash", step_id, "attention", detail) C.save_state(orch.state, args.runs_root, args.release) - C.emit(args.runs_root, args.release, f"[attention] signoffs: {detail}", kind="step") + C.emit(args.runs_root, args.release, f"[attention] {step_id}: {detail}", kind="step") print(detail) return 2 - orch.record_scout_step("bug_bash", "signoffs", "pass", - f"Sign-offs complete: Native Auth ({na}) + DID ({did})") - step = orch.state.get_step("bug_bash", "signoffs") + orch.record_scout_step("bug_bash", step_id, "pass", f"{label} sign-off: {by}") + step = orch.state.get_step("bug_bash", step_id) step.data = dict(step.data or {}) - step.data["native_auth_by"] = na - step.data["did_by"] = did + step.data["by"] = by step.by = "scout" - orch.state.set_step("bug_bash", "signoffs", step) + orch.state.set_step("bug_bash", step_id, step) C.save_state(orch.state, args.runs_root, args.release) - C.emit(args.runs_root, args.release, - f"[ok] signoffs: Native Auth ({na}) + DID ({did}) signed off", kind="step") - print(f"Recorded sign-offs: Native Auth={na}, DID={did}") + C.emit(args.runs_root, args.release, f"[ok] {step_id}: {label} sign-off ({by})", kind="step") + print(f"Recorded {label} sign-off: {by}") return 0 @@ -135,12 +135,16 @@ def register(sub): n.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") n.set_defaults(func=cmd_record_nativeauth_notify) - s = sub.add_parser("record-signoffs", - help="Record the bug-bash sign-offs (Native Auth attest + DID from Sowmya)") + s = sub.add_parser("record-native-auth-signoff", + help="Record the Native Auth sign-off (release engineer attests)") s.add_argument("--release", required=True) - s.add_argument("--native-auth-by", default=None, dest="native_auth_by", - help="Who signed off Native Auth. Omit (with --did-by) to hold the step.") - s.add_argument("--did-by", default=None, dest="did_by", - help="Who gave the DID sign-off. Omit to hold the step.") + s.add_argument("--by", default=None, help="Who signed off Native Auth. Omit to hold the step.") s.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") - s.set_defaults(func=cmd_record_signoffs) + s.set_defaults(func=cmd_record_native_auth_signoff) + + dd = sub.add_parser("record-did-signoff", + help="Record the DID sign-off (from the DID contact, Sowmya Malayanur)") + dd.add_argument("--release", required=True) + dd.add_argument("--by", default=None, help="Who gave the DID sign-off. Omit to hold the step.") + dd.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + dd.set_defaults(func=cmd_record_did_signoff) diff --git a/release-agent/steps/bug_bash/did_signoff.py b/release-agent/steps/bug_bash/did_signoff.py new file mode 100644 index 00000000..6952179e --- /dev/null +++ b/release-agent/steps/bug_bash/did_signoff.py @@ -0,0 +1,82 @@ +"""Step: `did_signoff` — reach out to the DID point of contact for the DID sign-off +(Phase 3, bug_bash). + +The second of the two bug-bash sign-offs (the first is `native_auth_signoff`). Scout sends a +1:1 Teams message to the DID point of contact (Sowmya Malayanur) asking for the DID sign-off, +then records it when it comes back. The step completes once the DID sign-off is captured. + +Scout-assisted: build() composes the DID request and instructs the skill to send it to the +DID contact, then record via `record-did-signoff --release --by ''` (or without +--by to hold the step). Idempotent — once recorded it reports done without re-sending. + +Depends on clone_plans_broker (for the test-plan link). +""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator.outcomes import NeedsSkill, Blocked, Done +from tools import testplans as T + +ID = "did_signoff" +KIND = "scout" + +# DID (Directory / Identity) sign-off point of contact. +DID_CONTACT = {"name": "Sowmya Malayanur", "email": "Sowmya.Malayanur@microsoft.com"} + + +def did_signed_by(state): + """Who gave the DID sign-off, or None.""" + return (state.get_step("bug_bash", ID).data or {}).get("by") + + +def _did_request_html(month_year, plan_url): + plan_line = (f' Test plan: {month_year} Broker test plan.' + if plan_url else "") + return ( + f'
    ' + f'

    Hi {DID_CONTACT["name"].split()[0]} \u2014 the {month_year} Android release ' + f'Bug Bash testing is complete on our side (Broker + Native Auth).

    ' + f'

    Could you please provide the DID sign-off when you get a chance?{plan_line}' + f' Thanks!

    ') + + +def build(state): + if not state.ccd: + return Blocked("did_signoff: no CCD set — can't identify the release month.") + + who = did_signed_by(state) + if who: + return Done(f"DID sign-off recorded for {state.release_id} ({who}).") + + month_year = schedule.parse_date(state.ccd).strftime("%B %Y") + broker_plan = (state.get_step("bug_bash", "clone_plans_broker").data or {}).get("plan_id") + plan_url = T.plan_web_url(broker_plan) if broker_plan else "" + html = _did_request_html(month_year, plan_url) + + instructions = ( + f"Request the DID sign-off for the {month_year} bug bash.\n" + f"1. Send the message below to the DID point of contact ({DID_CONTACT['name']}, " + f"{DID_CONTACT['email']}) as a 1:1 Teams message (`workiq_create_chat_by_email` " + f"\u2192 `workiq_send_chat_message`, contentType html).\n" + f"2. When the DID sign-off comes back, record it: `record-did-signoff --release " + f"{state.release_id} --by ''`. If it's still pending, run WITHOUT --by to hold " + f"the step (the request having been sent).") + + return NeedsSkill( + tool="record-did-signoff", + payload={ + "release": state.release_id, + "content": html, + "contentType": "html", + "did_contact": DID_CONTACT, + "followup_command": (f"record-did-signoff --release {state.release_id} " + f"--by ''"), + "_gather": {"month_year": month_year, "did_contact": DID_CONTACT, + "instructions": instructions}, + }, + record_as=ID, + summary=(f"Request the DID sign-off from {DID_CONTACT['name']} for the " + f"{month_year} bug bash"), + note="awaiting DID sign-off (from Sowmya Malayanur)", + outbound=True, + ) diff --git a/release-agent/steps/bug_bash/native_auth_signoff.py b/release-agent/steps/bug_bash/native_auth_signoff.py new file mode 100644 index 00000000..9a23d2e2 --- /dev/null +++ b/release-agent/steps/bug_bash/native_auth_signoff.py @@ -0,0 +1,56 @@ +"""Step: `native_auth_signoff` — the release engineer attests the Native Auth team finished +their tests and signed off (Phase 3, bug_bash). + +This is the first of the two bug-bash sign-offs (the other is `did_signoff`). No message is +sent — the release engineer simply confirms the Native Auth team completed their tests and +signed off (the confirmation they were asked to send back in `notify_native_auth`). The skill +captures who signed off. + +Scout-assisted: build() returns a NeedsSkill telling the skill to confirm the sign-off with +the release engineer and record it via `record-native-auth-signoff --release --by +''` (or without --by to hold the step until the sign-off arrives). Idempotent — once +recorded it reports done. +""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator.outcomes import NeedsSkill, Blocked, Done + +ID = "native_auth_signoff" +KIND = "scout" + + +def native_auth_signed_by(state): + """Who signed off Native Auth, or None.""" + return (state.get_step("bug_bash", ID).data or {}).get("by") + + +def build(state): + if not state.ccd: + return Blocked("native_auth_signoff: no CCD set — can't identify the release month.") + + who = native_auth_signed_by(state) + if who: + return Done(f"Native Auth sign-off recorded for {state.release_id} ({who}).") + + month_year = schedule.parse_date(state.ccd).strftime("%B %Y") + instructions = ( + f"Confirm the Native Auth sign-off for the {month_year} bug bash.\n" + f"1. Check with the release engineer that the Native Auth team finished their tests " + f"and signed off (this is the confirmation the Native Auth RE was asked to send back " + f"in notify_native_auth). Capture who signed off.\n" + f"2. Record it: `record-native-auth-signoff --release {state.release_id} --by " + f"''`. If it hasn't come in yet, run WITHOUT --by to hold the step.") + + return NeedsSkill( + tool="record-native-auth-signoff", + payload={ + "release": state.release_id, + "followup_command": (f"record-native-auth-signoff --release {state.release_id} " + f"--by ''"), + "_gather": {"month_year": month_year, "instructions": instructions}, + }, + record_as=ID, + summary=f"Attest the Native Auth team signed off for the {month_year} bug bash", + note="awaiting Native Auth sign-off (release engineer attests)", + ) diff --git a/release-agent/steps/bug_bash/signoffs.py b/release-agent/steps/bug_bash/signoffs.py deleted file mode 100644 index 2e4fbb73..00000000 --- a/release-agent/steps/bug_bash/signoffs.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Step: `signoffs` — gather the two release sign-offs (Phase 3, bug_bash). - -Two sign-offs close the bug bash before the owner's final attestation (bugbash_complete): - - 1. NATIVE AUTH — the release engineer attests the Native Auth team finished their tests and - signed off (the RE received their confirmation, per notify_native_auth). - 2. DID — reach out to the DID point of contact (Sowmya Malayanur) for the DID sign-off. - -Scout-assisted: build() composes the DID sign-off request and instructs the skill to send it -to the DID contact, then record both sign-offs. The step completes only when BOTH are -confirmed; otherwise it holds (the DID request having been sent). - -The skill records with `record-signoffs --release --native-auth-by '' --did-by -''` (both → done; missing either → hold for the owner). Idempotent — once both are -recorded it reports done without re-sending. -""" -from __future__ import annotations - -from orchestrator import schedule -from orchestrator.outcomes import NeedsSkill, Blocked, Done -from tools import testplans as T - -ID = "signoffs" -KIND = "scout" - -# DID (Directory / Identity) sign-off point of contact. -DID_CONTACT = {"name": "Sowmya Malayanur", "email": "Sowmya.Malayanur@microsoft.com"} - - -def recorded_signoffs(state): - """(native_auth_by, did_by) recorded on the step, each or None.""" - d = state.get_step("bug_bash", ID).data or {} - return (d.get("native_auth_by"), d.get("did_by")) - - -def _did_request_html(month_year, plan_url): - plan_line = (f' Test plan: {month_year} Broker test plan.' - if plan_url else "") - return ( - f'
    ' - f'

    Hi {DID_CONTACT["name"].split()[0]} \u2014 the {month_year} Android release ' - f'Bug Bash testing is complete on our side (Broker + Native Auth).

    ' - f'

    Could you please provide the DID sign-off when you get a chance?{plan_line}' - f' Thanks!

    ') - - -def build(state): - if not state.ccd: - return Blocked("signoffs: no CCD set — can't identify the release month.") - - na_by, did_by = recorded_signoffs(state) - if na_by and did_by: - return Done(f"Sign-offs complete for {state.release_id}: Native Auth ({na_by}) + " - f"DID ({did_by}).") - - month_year = schedule.parse_date(state.ccd).strftime("%B %Y") - broker_plan = (state.get_step("bug_bash", "clone_plans_broker").data or {}).get("plan_id") - plan_url = T.plan_web_url(broker_plan) if broker_plan else "" - html = _did_request_html(month_year, plan_url) - - instructions = ( - f"Gather the two {month_year} bug-bash sign-offs, then record them.\n" - f"1. NATIVE AUTH: confirm with the release engineer that the Native Auth team " - f"finished their tests and signed off (the RE should have received the confirmation " - f"requested in notify_native_auth). Capture who signed off.\n" - f"2. DID: send the message below to the DID point of contact " - f"({DID_CONTACT['name']}, {DID_CONTACT['email']}) as a 1:1 Teams message " - f"(`workiq_create_chat_by_email` \u2192 `workiq_send_chat_message`, contentType html), " - f"requesting the DID sign-off. Capture their sign-off when it comes back.\n" - f"3. Record when BOTH are in: `record-signoffs --release {state.release_id} " - f"--native-auth-by '' --did-by ''`. If either is still pending, run " - f"`record-signoffs --release {state.release_id}` (no flags) to hold the step.") - - return NeedsSkill( - tool="record-signoffs", - payload={ - "release": state.release_id, - "content": html, - "contentType": "html", - "did_contact": DID_CONTACT, - "followup_command": (f"record-signoffs --release {state.release_id} " - f"--native-auth-by '' --did-by ''"), - "_gather": {"month_year": month_year, "did_contact": DID_CONTACT, - "instructions": instructions}, - }, - record_as=ID, - summary=(f"Attest Native Auth sign-off + request DID sign-off from " - f"{DID_CONTACT['name']} for the {month_year} bug bash"), - note="awaiting sign-offs (Native Auth attest + DID from Sowmya Malayanur)", - outbound=True, - ) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 29263dc2..14e89f55 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -80,7 +80,8 @@ def _recent_iso(): "bug_bash.notify_native_auth": {"outcome": "done", "note": "native auth RE notified (test)"}, "bug_bash.activate_chat": {"outcome": "done", "note": "meeting chat activated (test)"}, "bug_bash.bugbash_updates": {"outcome": "done", "note": "first bug-bash update posted (test)"}, - "bug_bash.signoffs": {"outcome": "done", "note": "sign-offs recorded (test)"}, + "bug_bash.native_auth_signoff": {"outcome": "done", "note": "native auth sign-off recorded (test)"}, + "bug_bash.did_signoff": {"outcome": "done", "note": "DID sign-off recorded (test)"}, } @@ -646,8 +647,8 @@ def test_holds_at_first_hold(): # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/ # distribute_tests (3) + send_invite + activate_chat (scout, mocked done) (2) + - # notify_native_auth (1) + bugbash_updates (1) + signoffs (scout, mocked done) (1). - assert sum(1 for a in actions if a.kind == "ran") == 17 + # notify_native_auth (1) + bugbash_updates (1) + native_auth_signoff + did_signoff (2). + assert sum(1 for a in actions if a.kind == "ran") == 18 def test_gate_blocks_until_approved(): @@ -3070,62 +3071,70 @@ def test_record_nativeauth_notify_stores_or_holds(): assert after.get_step("bug_bash", "notify_native_auth").status == "blocked" -# ---- Phase 3: signoffs (Native Auth attest + DID) ---- +# ---- Phase 3: native_auth_signoff + did_signoff ---- -def test_signoffs_composes_needs_skill_with_did_request(): - """signoffs is a scout step: NeedsSkill composing the DID sign-off request to Sowmya + - instructions to gather the Native Auth attest, with the recorder follow-up.""" +def test_native_auth_signoff_composes_and_idempotent(): + """native_auth_signoff is a scout step that attests the Native Auth sign-off (no outbound + message); once recorded it's idempotent-done.""" import steps as _steps from orchestrator.outcomes import as_dict - from steps.bug_bash.signoffs import DID_CONTACT + from orchestrator.state import StepState st = _na_state() - out = as_dict(_steps.get_step("bug_bash", "signoffs").build(st)) - assert out["kind"] == "needs_skill" and out["tool"] == "record-signoffs" - assert out["payload"]["did_contact"]["email"] == "Sowmya.Malayanur@microsoft.com" - assert DID_CONTACT["name"].split()[0] in out["payload"]["content"] # greets Sowmya - assert "DID sign-off" in out["payload"]["content"] - assert "planId=3730001" in out["payload"]["content"] # links the plan - assert "record-signoffs --release 2026-08" in out["payload"]["followup_command"] - assert out["outbound"] is True - assert _steps.get_step("bug_bash", "signoffs").KIND == "scout" - - -def test_signoffs_idempotent_when_both_recorded(): + out = as_dict(_steps.get_step("bug_bash", "native_auth_signoff").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "record-native-auth-signoff" + assert "record-native-auth-signoff --release 2026-08" in out["payload"]["followup_command"] + assert not out.get("outbound") # attestation, nothing sent + assert _steps.get_step("bug_bash", "native_auth_signoff").KIND == "scout" + st.set_step("bug_bash", "native_auth_signoff", StepState(status="done", data={"by": "silviu"})) + out2 = as_dict(_steps.get_step("bug_bash", "native_auth_signoff").build(st)) + assert out2["kind"] == "done" and "silviu" in out2["note"] + + +def test_did_signoff_composes_did_request_and_idempotent(): + """did_signoff is a scout step: NeedsSkill composing the DID request to Sowmya (outbound), + linking the plan; once recorded it's idempotent-done.""" import steps as _steps from orchestrator.outcomes import as_dict from orchestrator.state import StepState + from steps.bug_bash.did_signoff import DID_CONTACT st = _na_state() - st.set_step("bug_bash", "signoffs", - StepState(status="done", data={"native_auth_by": "silviu", "did_by": "sowmya"})) - out = as_dict(_steps.get_step("bug_bash", "signoffs").build(st)) - assert out["kind"] == "done" and "Sign-offs complete" in out["note"] + out = as_dict(_steps.get_step("bug_bash", "did_signoff").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "record-did-signoff" + assert out["payload"]["did_contact"]["email"] == "Sowmya.Malayanur@microsoft.com" + assert DID_CONTACT["name"].split()[0] in out["payload"]["content"] # greets Sowmya + assert "DID sign-off" in out["payload"]["content"] and "planId=3730001" in out["payload"]["content"] + assert out["outbound"] is True + st.set_step("bug_bash", "did_signoff", StepState(status="done", data={"by": "sowmya"})) + out2 = as_dict(_steps.get_step("bug_bash", "did_signoff").build(st)) + assert out2["kind"] == "done" and "sowmya" in out2["note"] -def test_record_signoffs_requires_both(): - """record-signoffs marks done only when BOTH sign-offs are given; missing either holds.""" +def test_record_signoff_commands_store_or_hold(): + """record-native-auth-signoff / record-did-signoff store --by + mark done; omitting --by + holds the step for the owner.""" import tempfile, argparse from orchestrator import cli_common as _C from orchestrator.commands import bugbash_chat as BC - from steps.bug_bash.signoffs import recorded_signoffs + from steps.bug_bash.native_auth_signoff import native_auth_signed_by + from steps.bug_bash.did_signoff import did_signed_by with tempfile.TemporaryDirectory() as d: rid = "2026-08" _stub_build_defs("pass") - # only one → hold _C.save_state(_na_state(), d, rid) - ns = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, - native_auth_by="silviu", did_by=None) - assert BC.cmd_record_signoffs(ns) == 2 + # native auth: with --by → done + ns = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, by="silviu") + assert BC.cmd_record_native_auth_signoff(ns) == 0 + assert native_auth_signed_by(_C.load_state(d, rid)) == "silviu" + # DID: without --by → hold + ns2 = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, by=None) + assert BC.cmd_record_did_signoff(ns2) == 2 held = _C.load_state(d, rid) - assert not held.is_done("bug_bash", "signoffs") - assert held.get_step("bug_bash", "signoffs").status == "blocked" - - # both → done - ns2 = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, - native_auth_by="silviu", did_by="sowmya") - assert BC.cmd_record_signoffs(ns2) == 0 - done = _C.load_state(d, rid) - assert done.is_done("bug_bash", "signoffs") - assert recorded_signoffs(done) == ("silviu", "sowmya") + assert not held.is_done("bug_bash", "did_signoff") + assert held.get_step("bug_bash", "did_signoff").status == "blocked" + # DID: with --by → done + ns3 = argparse.Namespace(runs_root=d, release=rid, config=CONFIG, as_of=None, by="sowmya") + assert BC.cmd_record_did_signoff(ns3) == 0 + assert did_signed_by(_C.load_state(d, rid)) == "sowmya" # ---- Phase 3: bugbash_updates (periodic poster) ----