diff --git a/.gitattributes b/.gitattributes index 58fcbf0b0..c9c2ed74c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,9 +1,22 @@ -# Don't allow people to merge changes to these generated files, because the result -# may be invalid. You need to run "rush update" again. +# Lockfiles for package managers we don't use must never be text-merged: a bad +# merge silently corrupts them, so keep them as hard conflicts. shrinkwrap.yaml merge=binary npm-shrinkwrap.json merge=binary yarn.lock merge=binary -pnpm-lock.yaml merge=binary + +# pnpm-lock.yaml is regenerated on merge by a local driver that ships with the +# repo (registered by ts/tools/scripts/setup-merge-driver.mjs on `pnpm install`). +# Hosts without the driver (e.g. GitHub's server-side merges and the merge +# queue) don't recognize the name and fall back to git's normal 3-way text +# merge; CI's `pnpm install --frozen-lockfile` then catches any bad merge. +pnpm-lock.yaml merge=pnpm-lock + +# Autogenerated package READMEs carry a commit-stamped footer, so every branch +# regenerates a different last line and they conflict constantly. Keep our copy +# on merge (the local keep-ours driver); the docs pipeline regenerates them. +# Where the driver isn't registered the name is unknown and git falls back to a +# text merge, and the heal-generated-files workflow clears the leftover conflict. +README.AUTOGEN.md merge=keep-ours # Make text consistently using LF * text eol=lf diff --git a/.github/workflows/heal-generated-files.yml b/.github/workflows/heal-generated-files.yml new file mode 100644 index 000000000..2e422f2ee --- /dev/null +++ b/.github/workflows/heal-generated-files.yml @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Auto-heal generated-file conflicts on open pull requests. When main moves, +# open PRs can start conflicting on generated files - the pnpm lockfile and the +# autogenerated README.AUTOGEN.md companions - and get ejected from the merge +# queue. This job merges main into each affected same-repo PR branch and +# resolves those files so no one has to do it by hand. The logic lives in +# ts/tools/scripts/heal-generated-files.mjs. + +name: Heal Generated Files + +on: + push: + branches: ["main"] + workflow_dispatch: + inputs: + pr: + description: "Single PR number to heal (default: scan all open PRs)" + type: string + +concurrency: + # Serialize runs so two never push to the same branch at once, and never + # cancel a run mid-push. + group: heal-generated-files + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + heal: + runs-on: ubuntu-latest + env: + # `vars`/`secrets` cannot be used directly in step `if` conditions, so + # expose whether the app credentials are configured as a string. The App + # id is a repo variable (as in docs-pr-amend.yml / fix-dependabot-alerts). + HAS_APP_CREDS: ${{ vars.DEPENDABOT_APP_ID != '' }} + steps: + - name: Setup Git LF + run: git config --global core.autocrlf false + + # Push to PR branches with the typeagent-bot app token (not the default + # GITHUB_TOKEN), so the healing commit re-triggers the PR's normal checks. + - name: Generate app token + id: app-token + if: env.HAS_APP_CREDS == 'true' + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.DEPENDABOT_APP_ID }} + private-key: ${{ secrets.DEPENDABOT_APP_PRIVATE_KEY }} + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }} + + - uses: pnpm/action-setup@v4 + name: Install pnpm + with: + package_json_file: ts/package.json + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: "pnpm" + cache-dependency-path: ts/pnpm-lock.yaml + + - name: Heal conflicted generated files + if: env.HAS_APP_CREDS == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + INPUT_PR: ${{ github.event.inputs.pr }} + run: node ts/tools/scripts/heal-generated-files.mjs diff --git a/ts/package.json b/ts/package.json index 8d7793408..f0ece7265 100644 --- a/ts/package.json +++ b/ts/package.json @@ -46,7 +46,7 @@ "getKeys": "node tools/scripts/getKeys.mjs", "getKeys:build": "node tools/scripts/getKeys.mjs --vault build-pipeline-kv", "getNPMRC": "node tools/scripts/getNPMRC.mjs", - "postinstall": "pnpm run postinstall:better-sqlite3-node-copy && pnpm run postinstall:better-sqlite3-node-restore", + "postinstall": "pnpm run postinstall:better-sqlite3-node-copy && pnpm run postinstall:better-sqlite3-node-restore && pnpm run postinstall:merge-driver", "install:ext:vscode": "fluid-build . -t install:ext:vscode", "knowledgeVisualizer": "pnpm -C packages/knowledgeVisualizer exec npm run start", "kv": "pnpm -C packages/knowledgeVisualizer exec npm run start", @@ -56,6 +56,7 @@ "montage": "pnpm -C packages/montage exec npm run start", "postinstall:better-sqlite3-node-copy": "node ./tools/scripts/copy-better-sqlite3-node.js", "postinstall:better-sqlite3-node-restore": "node ./tools/scripts/restore-better-sqlite3-node.js", + "postinstall:merge-driver": "node tools/scripts/setup-merge-driver.mjs", "prettier": "prettier --check .", "prettier:changed": "node tools/scripts/prettier-changed.mjs", "prettier:changed:fix": "node tools/scripts/prettier-changed.mjs --write", diff --git a/ts/tools/scripts/getKeys.mjs b/ts/tools/scripts/getKeys.mjs index fd3c7d138..e1c5be4db 100644 --- a/ts/tools/scripts/getKeys.mjs +++ b/ts/tools/scripts/getKeys.mjs @@ -109,6 +109,26 @@ async function getPIMClient() { return getClient(); } +// Self-activate a PIM role for a short window so the following data-plane +// operation can succeed. Throws if PIM is unavailable or activation fails, so +// callers can fall back to another role or an unelevated retry. +async function elevate(roleName) { + console.warn(chalk.yellowBright(`Elevating to '${roleName}'...`)); + const pimClient = await getPIMClient(); + await pimClient.elevate({ + requestType: "SelfActivate", + roleName, + expirationType: "AfterDuration", + expirationDuration: "PT5M", // activate for 5 minutes + continueOnFailure: true, + }); + + // Wait for the role to be activated + console.log(chalk.green("Elevation successful.")); + console.warn(chalk.yellowBright("Waiting 5 seconds...")); + await new Promise((res) => setTimeout(res, 5000)); +} + async function getSecretListWithElevation(keyVaultClient, vaultName) { try { return await keyVaultClient.getSecrets(vaultName); @@ -118,23 +138,9 @@ async function getSecretListWithElevation(keyVaultClient, vaultName) { } try { - console.warn(chalk.yellowBright("Elevating to get secrets...")); - const pimClient = await getPIMClient(); - await pimClient.elevate({ - requestType: "SelfActivate", - roleName: "Key Vault Administrator", - expirationType: "AfterDuration", - expirationDuration: "PT5M", // activate for 5 minutes - continueOnFailure: true, - }); - - // Wait for the role to be activated - console.log(chalk.green("Elevation successful.")); - console.warn(chalk.yellowBright("Waiting 5 seconds...")); - await new Promise((res) => setTimeout(res, 5000)); - + await elevate("Key Vault Administrator"); return await keyVaultClient.getSecrets(vaultName); - } catch (e) { + } catch { console.warn( chalk.yellow( "Elevation to key vault admin failed...attempting to get secrets as key vault reader.", @@ -143,21 +149,8 @@ async function getSecretListWithElevation(keyVaultClient, vaultName) { } try { - console.warn(chalk.yellowBright("Elevating to get secrets...")); - const pimClient = await getPIMClient(); - await pimClient.elevate({ - requestType: "SelfActivate", - roleName: "Key Vault Secrets User", - expirationType: "AfterDuration", - expirationDuration: "PT5M", // activate for 5 minutes - continueOnFailure: true, - }); - - // Wait for the role to be activated - console.log(chalk.green("Elevation successful.")); - console.warn(chalk.yellowBright("Waiting 5 seconds...")); - await new Promise((res) => setTimeout(res, 5000)); - } catch (e) { + await elevate("Key Vault Secrets User"); + } catch { console.warn( chalk.yellow( "Elevation failed...attempting to get secrets without elevation.", @@ -169,6 +162,61 @@ async function getSecretListWithElevation(keyVaultClient, vaultName) { } } +// Write a secret, self-elevating via PIM if the vault rejects the write with a +// 403 (the signed-in user lacks STANDING write access). Mirrors +// getSecretListWithElevation, but targets roles that grant write (set) +// permission: "Key Vault Secrets User" is read-only and useless here, so we try +// "Key Vault Administrator" then "Key Vault Secrets Officer". +async function writeSecretWithElevation( + keyVaultClient, + vaultName, + secretName, + secretValue, +) { + try { + return await keyVaultClient.writeSecret( + vaultName, + secretName, + secretValue, + ); + } catch (e) { + if (!isForbiddenByRbac(e)) { + throw e; + } + + try { + await elevate("Key Vault Administrator"); + return await keyVaultClient.writeSecret( + vaultName, + secretName, + secretValue, + ); + } catch { + console.warn( + chalk.yellow( + "Elevation to key vault admin failed...trying key vault secrets officer.", + ), + ); + } + + try { + await elevate("Key Vault Secrets Officer"); + } catch { + console.warn( + chalk.yellow( + "Elevation failed...attempting to write without elevation.", + ), + ); + } + + return await keyVaultClient.writeSecret( + vaultName, + secretName, + secretValue, + ); + } +} + async function getSecrets(keyVaultClient, vaultName, shared) { const overallStart = Date.now(); console.log( @@ -576,7 +624,12 @@ async function pushYamlConfig() { } try { - await keyVaultClient.writeSecret(vaultName, secretName, yamlContent); + await writeSecretWithElevation( + keyVaultClient, + vaultName, + secretName, + yamlContent, + ); console.log( chalk.green( `\nSecret '${secretName}' updated in vault '${vaultName}'.`, diff --git a/ts/tools/scripts/heal-generated-files.mjs b/ts/tools/scripts/heal-generated-files.mjs new file mode 100644 index 000000000..3cab404cb --- /dev/null +++ b/ts/tools/scripts/heal-generated-files.mjs @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Auto-heal generated-file conflicts on open pull requests. Run by the +// heal-generated-files workflow after main moves (or on manual dispatch). For each +// same-repo PR that conflicts with main ONLY in generated files - the pnpm +// lockfile and/or the autogenerated README.AUTOGEN.md companions - merge main +// into the PR branch, resolve those files, and push the result so the PR (and +// the merge queue) stops choking on them. The lockfile is rebuilt with pnpm; +// the READMEs keep our copy and are regenerated by the docs-pr-amend workflow +// that re-runs on the healed push. PRs that conflict in any other file are +// left untouched for a human to resolve. + +import { execFileSync } from "node:child_process"; + +const LOCKFILE = "ts/pnpm-lock.yaml"; +const REPO = process.env.GITHUB_REPOSITORY; +const ONLY_PR = process.env.INPUT_PR?.trim(); + +// All git/gh/pnpm calls go through execFileSync with an argument array (no +// shell), so branch names coming from the API can never be interpreted by a +// shell. +function run(cmd, args, opts = {}) { + return execFileSync(cmd, args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + ...opts, + }); +} + +function tryRun(cmd, args, opts = {}) { + try { + return { ok: true, out: run(cmd, args, opts) }; + } catch (err) { + return { ok: false, out: `${err.stdout ?? ""}${err.stderr ?? ""}` }; + } +} + +function candidatePRs() { + if (ONLY_PR) { + const p = JSON.parse( + run("gh", [ + "pr", + "view", + ONLY_PR, + "--repo", + REPO, + "--json", + "number,headRefName,isCrossRepository", + ]), + ); + if (p.isCrossRepository) { + console.log( + `[heal] PR #${ONLY_PR} is from a fork - cannot push, skipping`, + ); + return []; + } + return [{ number: p.number, ref: p.headRefName }]; + } + // Open PRs targeting main, from this repo (fork branches aren't pushable), + // that aren't already cleanly mergeable. A trial merge decides the rest. + const prs = JSON.parse( + run("gh", [ + "pr", + "list", + "--repo", + REPO, + "--state", + "open", + "--base", + "main", + "--limit", + "100", + "--json", + "number,mergeable,isCrossRepository,headRefName", + ]), + ); + return prs + .filter((p) => !p.isCrossRepository && p.mergeable !== "MERGEABLE") + .map((p) => ({ number: p.number, ref: p.headRefName })); +} + +function unmergedFiles() { + return run("git", ["diff", "--name-only", "--diff-filter=U"]) + .split("\n") + .map((s) => s.trim()) + .filter(Boolean); +} + +// Generated files we can resolve automatically after merging main. A conflict +// in anything else means a human needs to look at the PR. +function isAutogenReadme(file) { + return /^ts\/packages\/.+\/README\.AUTOGEN\.md$/.test(file); +} + +function isResolvable(file) { + return file === LOCKFILE || isAutogenReadme(file); +} + +// Resolve the merged-in conflicts in place and stage the results. +function resolveConflicts(conflicts) { + // Autogen READMEs: keep our copy. The docs-pr-amend workflow regenerates + // them from the merged source on the healed push (it re-runs because we push + // with the app token), so no build or model access is needed here. + for (const file of conflicts.filter(isAutogenReadme)) { + run("git", ["checkout", "--ours", "--", file]); + run("git", ["add", file]); + } + + // Lockfile: seed from main's copy, then let pnpm rebuild it against the + // merged package.json files. + if (conflicts.includes(LOCKFILE)) { + run("git", ["checkout", "--theirs", "--", LOCKFILE]); + run( + "pnpm", + [ + "install", + "--lockfile-only", + "--no-frozen-lockfile", + "--ignore-scripts", + ], + { cwd: "ts", stdio: "inherit" }, + ); + run("git", ["add", LOCKFILE]); + } +} + +function healPr({ number, ref }) { + console.log(`\n[heal] PR #${number} (${ref})`); + try { + run("git", ["fetch", "origin", "main", ref]); + run("git", ["checkout", "-B", "_heal", `origin/${ref}`]); + + // Trial-merge main. --no-ff/--no-commit so a clean merge leaves a + // MERGE_HEAD we simply discard, and a conflicted one leaves the unmerged + // set to inspect. + tryRun("git", ["merge", "--no-ff", "--no-commit", "origin/main"]); + const conflicts = unmergedFiles(); + + if (conflicts.length === 0) { + console.log("[heal] no conflict - skipping"); + return; + } + if (!conflicts.every(isResolvable)) { + const other = conflicts.filter((f) => !isResolvable(f)); + console.log( + `[heal] conflicts beyond generated files (${other.join(", ")}) - leaving for a human`, + ); + return; + } + + resolveConflicts(conflicts); + run("git", ["commit", "--no-edit"]); + + // Non-force push: if the PR head moved meanwhile the push is rejected and + // the next run retries, so a concurrent update is never clobbered. + const push = tryRun("git", ["push", "origin", `_heal:${ref}`]); + if (push.ok) { + console.log( + `[heal] PR #${number}: resolved ${conflicts.join(", ")} and pushed`, + ); + } else { + console.log( + `[heal] PR #${number}: push rejected (branch moved?) - will retry\n${push.out}`, + ); + } + } finally { + // Return to a clean, detached state so the next PR starts fresh. This + // only touches the ephemeral CI checkout and the throwaway _heal branch. + tryRun("git", ["merge", "--abort"]); + tryRun("git", ["reset", "--hard"]); + tryRun("git", ["checkout", "--detach", "origin/main"]); + tryRun("git", ["branch", "-D", "_heal"]); + } +} + +const prs = candidatePRs(); +if (prs.length === 0) { + console.log("[heal] nothing to do - no conflicting same-repo PRs"); + process.exit(0); +} + +run("git", ["config", "user.name", "typeagent-bot"]); +run("git", [ + "config", + "user.email", + "typeagent-bot[bot]@users.noreply.github.com", +]); + +let failures = 0; +for (const pr of prs) { + try { + healPr(pr); + } catch (err) { + failures++; + console.error(`[heal] PR #${pr.number} failed: ${err.message}`); + } +} +process.exit(failures ? 1 : 0); diff --git a/ts/tools/scripts/merge-pnpm-lock.mjs b/ts/tools/scripts/merge-pnpm-lock.mjs new file mode 100644 index 000000000..d37b3e349 --- /dev/null +++ b/ts/tools/scripts/merge-pnpm-lock.mjs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Git merge driver for pnpm-lock.yaml. A line-level 3-way merge of a lockfile +// can silently produce an invalid result, so instead we keep our side and let +// pnpm rebuild a correct lockfile from the already-merged package.json files. +// +// git invokes this as: node merge-pnpm-lock.mjs %O %A %B %P +// %O = common ancestor, %A = ours (git reads the merged result back from it), +// %B = theirs, %P = path of the lockfile in the work tree (ts/pnpm-lock.yaml). + +import { execSync } from "node:child_process"; +import { copyFileSync } from "node:fs"; +import { dirname } from "node:path"; + +const ours = process.argv[3]; // %A +const lockfilePath = process.argv[5]; // %P + +try { + // Seed with our lockfile, then let pnpm reconcile it to the merged manifests. + copyFileSync(ours, lockfilePath); + execSync( + "pnpm install --lockfile-only --no-frozen-lockfile --ignore-scripts", + { cwd: dirname(lockfilePath), stdio: "inherit" }, + ); + copyFileSync(lockfilePath, ours); + process.exit(0); +} catch (err) { + console.error(`[merge-pnpm-lock] regeneration failed: ${err.message}`); + process.exit(1); // non-zero leaves the conflict for manual resolution +} diff --git a/ts/tools/scripts/setup-merge-driver.mjs b/ts/tools/scripts/setup-merge-driver.mjs new file mode 100644 index 000000000..19dffbab1 --- /dev/null +++ b/ts/tools/scripts/setup-merge-driver.mjs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Registers local git merge drivers for generated files that conflict often: +// - pnpm-lock.yaml: rebuilt from the merged manifests (merge-pnpm-lock.mjs). +// - README.AUTOGEN.md: keep our copy (the docs pipeline regenerates them). +// Runs from the root package's postinstall on every `pnpm install`, so every +// clone gets the drivers without any manual setup. Does nothing outside a git +// work tree (CI tarballs, Docker image builds) and never fails the install. + +import { execFileSync } from "node:child_process"; + +function git(args) { + execFileSync("git", args, { stdio: "ignore" }); +} + +// The merge attribute (pnpm-lock.yaml merge=pnpm-lock) lives in .gitattributes +// and ships with the repo; only the driver command is per-clone local config, +// so it is the one piece that has to be registered here. +const DRIVER = "node ts/tools/scripts/merge-pnpm-lock.mjs %O %A %B %P"; + +try { + git(["rev-parse", "--is-inside-work-tree"]); +} catch { + process.exit(0); // not a git checkout - nothing to register +} + +try { + git([ + "config", + "merge.pnpm-lock.name", + "Regenerate pnpm-lock.yaml on merge", + ]); + git(["config", "merge.pnpm-lock.driver", DRIVER]); + + // Autogenerated READMEs carry a commit-stamped footer, so every branch + // regenerates a different last line and they conflict constantly. Keep our + // side on merge; `true` is git's documented "keep ours" driver, and the docs + // pipeline regenerates the file afterward. + git([ + "config", + "merge.keep-ours.name", + "Keep our copy (regenerated by the docs pipeline)", + ]); + git(["config", "merge.keep-ours.driver", "true"]); +} catch (err) { + // A missing or locked git config must never break `pnpm install`. + console.warn( + `[setup-merge-driver] could not register driver: ${err.message}`, + ); +}