From 69c1b3316fc794244c4694c8651267aa2f7c60e8 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Fri, 17 Jul 2026 11:15:47 -0700 Subject: [PATCH 1/4] Mirror Node SDK releases internally Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b6e76edd-7acc-4681-a884-56d16967c048 --- .github/workflows/publish.yml | 154 ++++++++++++++++++-- nodejs/scripts/npm-release.js | 248 ++++++++++++++++++++++++++++++++ nodejs/test/npm-release.test.ts | 246 +++++++++++++++++++++++++++++++ 3 files changed, 635 insertions(+), 13 deletions(-) create mode 100644 nodejs/scripts/npm-release.js create mode 100644 nodejs/test/npm-release.test.ts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cd96dd0fa9..e6eb820c8b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,9 +21,7 @@ on: required: false permissions: - contents: write - id-token: write # Required for OIDC - actions: write # Required to trigger changelog workflow + contents: read concurrency: group: publish @@ -78,11 +76,21 @@ jobs: echo "Auto-incremented version: $VERSION" >> $GITHUB_STEP_SUMMARY fi echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + - name: Verify version is available on public npm + env: + VERSION: ${{ steps.version.outputs.VERSION }} + run: | + node scripts/npm-release.js preflight \ + --package @github/copilot-sdk \ + --version "$VERSION" \ + --registry https://registry.npmjs.org - publish-nodejs: - name: Publish Node.js SDK + package-nodejs: + name: Package Node.js SDK needs: version runs-on: ubuntu-latest + permissions: + contents: read defaults: run: working-directory: ./nodejs @@ -91,8 +99,6 @@ jobs: - uses: actions/setup-node@v6 with: node-version: "22.x" - - name: Update npm for OIDC support - run: npm i -g "npm@11.6.3" - run: npm ci --ignore-scripts - name: Set version run: node scripts/set-version.js @@ -101,21 +107,126 @@ jobs: - name: Build run: npm run build - name: Pack - run: npm pack + id: pack + run: | + TARBALL="$(npm pack . --json | jq -r '.[0].filename')" + if [ -z "$TARBALL" ] || [ ! -f "$TARBALL" ]; then + echo "::error::npm pack did not produce a tarball." + exit 1 + fi + echo "tarball=$TARBALL" >> "$GITHUB_OUTPUT" - name: Upload artifact uses: actions/upload-artifact@v7.0.0 with: name: nodejs-package - path: nodejs/*.tgz - - name: Publish to npm - if: github.ref == 'refs/heads/main' || github.event.inputs.dist-tag == 'unstable' - run: npm publish --tag ${{ github.event.inputs.dist-tag }} --access public --registry https://registry.npmjs.org + path: nodejs/${{ steps.pack.outputs.tarball }} + if-no-files-found: error + + publish-nodejs: + name: Publish Node.js SDK + needs: package-nodejs + if: github.ref == 'refs/heads/main' || github.event.inputs.dist-tag == 'unstable' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + id-token: write + steps: + - uses: actions/checkout@v6.0.2 + - uses: actions/setup-node@v6 + with: + node-version: "22.x" + - name: Update npm for OIDC support + run: npm i -g "npm@11.6.3" + - name: Download Node.js package + uses: actions/download-artifact@v8.0.0 + with: + name: nodejs-package + path: ./dist + - name: Publish tarball to public npm + env: + DIST_TAG: ${{ github.event.inputs.dist-tag }} + run: | + set -euo pipefail + shopt -s nullglob + TARBALLS=(./dist/*.tgz) + if [ "${#TARBALLS[@]}" -ne 1 ]; then + echo "::error::Expected exactly one Node.js package tarball, found ${#TARBALLS[@]}." + exit 1 + fi + node nodejs/scripts/npm-release.js publish \ + --tarball "${TARBALLS[0]}" \ + --tag "$DIST_TAG" \ + --access public \ + --registry https://registry.npmjs.org + + publish-nodejs-internal: + name: Publish Node.js SDK to internal feed + needs: publish-nodejs + environment: cicd + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + id-token: write + env: + ADO_RESOURCE: 499b84ac-1321-427f-aa17-267ca6975798 + FEED_URL: https://pkgs.dev.azure.com/devdiv/_packaging/copilot-canary/npm/registry/ + steps: + - uses: actions/checkout@v6.0.2 + - uses: actions/setup-node@v6 + with: + node-version: "22.x" + - name: Download Node.js package + uses: actions/download-artifact@v8.0.0 + with: + name: nodejs-package + path: ./dist + - name: Azure Login (OIDC -> id-cpd-ci) + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 + with: + client-id: "${{ vars.CPD_ID_CLIENT_ID }}" # id-cpd-ci + tenant-id: "${{ vars.CPD_ID_TENANT_ID }}" + allow-no-subscriptions: true + - name: Configure feed auth + run: | + set -euo pipefail + TOKEN="$(az account get-access-token --resource "$ADO_RESOURCE" --query accessToken -o tsv)" + echo "::add-mask::$TOKEN" + FEED_AUTH_REGISTRY="${FEED_URL#https:}" + FEED_AUTH_BASE="${FEED_AUTH_REGISTRY%registry/}" + printf '%s\n' \ + "${FEED_AUTH_REGISTRY}:_authToken=${TOKEN}" \ + "${FEED_AUTH_BASE}:_authToken=${TOKEN}" > "$HOME/.npmrc" + - name: Publish tarball to internal feed + env: + DIST_TAG: ${{ github.event.inputs.dist-tag }} + run: | + set -euo pipefail + if [ "$FEED_URL" != "https://pkgs.dev.azure.com/devdiv/_packaging/copilot-canary/npm/registry/" ]; then + echo "::error::FEED_URL ('$FEED_URL') is not the expected internal feed. Refusing to publish." + exit 1 + fi + shopt -s nullglob + TARBALLS=(./dist/*.tgz) + if [ "${#TARBALLS[@]}" -ne 1 ]; then + echo "::error::Expected exactly one Node.js package tarball, found ${#TARBALLS[@]}." + exit 1 + fi + node nodejs/scripts/npm-release.js publish \ + --tarball "${TARBALLS[0]}" \ + --tag "$DIST_TAG" \ + --registry "$FEED_URL" \ + --azure true publish-dotnet: name: Publish .NET SDK if: github.event.inputs.dist-tag != 'unstable' needs: version runs-on: ubuntu-latest + permissions: + contents: read + id-token: write defaults: run: working-directory: ./dotnet @@ -200,6 +311,9 @@ jobs: if: github.event.inputs.dist-tag != 'unstable' needs: version runs-on: ubuntu-latest + permissions: + contents: read + id-token: write defaults: run: working-directory: ./python @@ -237,6 +351,9 @@ jobs: name: Publish Java SDK if: github.event.inputs.dist-tag != 'unstable' && github.ref == 'refs/heads/main' needs: version + permissions: + contents: write + id-token: write uses: ./.github/workflows/java-publish-maven.yml with: releaseVersion: ${{ needs.version.outputs.version }} @@ -245,7 +362,15 @@ jobs: github-release: name: Create GitHub Release - needs: [version, publish-nodejs, publish-dotnet, publish-python, publish-rust, publish-java] + needs: + [ + version, + publish-nodejs, + publish-dotnet, + publish-python, + publish-rust, + publish-java, + ] if: | always() && github.ref == 'refs/heads/main' && @@ -256,6 +381,9 @@ jobs: needs.publish-python.result == 'success' && needs.publish-rust.result == 'success' runs-on: ubuntu-latest + permissions: + actions: write + contents: write steps: - uses: actions/checkout@v6.0.2 - name: Create GitHub Release diff --git a/nodejs/scripts/npm-release.js b/nodejs/scripts/npm-release.js new file mode 100644 index 0000000000..73c3967469 --- /dev/null +++ b/nodejs/scripts/npm-release.js @@ -0,0 +1,248 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +const PUBLIC_CONFLICT = + /^(?:npm (?:error|ERR!) code EPUBLISHCONFLICT|npm (?:error|ERR!) (?:403 [^\r\n]* - )?(?:You )?cannot publish over (?:the )?previously published versions(?:: [^\r\n]+)?\.?)\r?$/im; +const AZURE_TARBALL_CONFLICT = + /^npm (?:error|ERR!) (?:403 [^\r\n]* - )?already contains file '[^'\r\n]+\.tgz' in package '[^'\r\n]+'\.?\r?$/im; +const DEFAULT_RETRY_DELAYS_MS = [0, 1000, 2000, 4000, 8000]; + +export async function runCommand(command, args, { stream = false } = {}) { + return await new Promise((resolve, reject) => { + const child = spawn(command, args, { shell: false }); + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (chunk) => { + const text = chunk.toString(); + stdout += text; + if (stream) { + process.stdout.write(chunk); + } + }); + child.stderr.on("data", (chunk) => { + const text = chunk.toString(); + stderr += text; + if (stream) { + process.stderr.write(chunk); + } + }); + child.on("error", reject); + child.on("close", (status) => resolve({ status: status ?? 1, stdout, stderr })); + }); +} + +export async function assertVersionAbsent({ packageName, version, registry, runner = runCommand }) { + const result = await runner("npm", [ + "view", + `${packageName}@${version}`, + "version", + "--json", + "--registry", + registry, + ]); + const output = `${result.stdout}\n${result.stderr}`; + + if (result.status === 0) { + let publishedVersion; + try { + publishedVersion = JSON.parse(result.stdout); + } catch { + throw new Error( + `Public npm returned malformed version metadata for ${packageName}@${version}.` + ); + } + if (publishedVersion !== version) { + throw new Error( + `Public npm returned unexpected version metadata for ${packageName}@${version}: ${result.stdout.trim()}` + ); + } + throw new Error(`${packageName}@${version} already exists on public npm.`); + } + + try { + const errorResult = JSON.parse(result.stdout); + if (errorResult?.error?.code === "E404") { + return; + } + } catch { + // The failure below includes npm's output for diagnosis. + } + + throw new Error( + `Could not confirm that ${packageName}@${version} is absent from public npm (npm exited ${result.status}).\n${output.trim()}` + ); +} + +export async function inspectTarball(tarball, runner = runCommand) { + const result = await runner("tar", ["-xOf", tarball, "package/package.json"]); + if (result.status !== 0) { + throw new Error( + `Could not read package/package.json from ${tarball}.\n${result.stderr.trim()}` + ); + } + + let manifest; + try { + manifest = JSON.parse(result.stdout); + } catch { + throw new Error(`Tarball ${tarball} contains malformed package metadata.`); + } + if (typeof manifest.name !== "string" || typeof manifest.version !== "string") { + throw new Error(`Tarball ${tarball} does not contain a valid package name and version.`); + } + + const bytes = await readFile(tarball); + const integrity = `sha512-${createHash("sha512").update(bytes).digest("base64")}`; + return { name: manifest.name, version: manifest.version, integrity }; +} + +function parseIntegrity(stdout) { + try { + const integrity = JSON.parse(stdout); + if (typeof integrity !== "string") { + return null; + } + const match = /^sha512-([A-Za-z0-9+/]+={0,2})$/.exec(integrity); + return match && Buffer.from(match[1], "base64").length === 64 ? integrity : null; + } catch { + return null; + } +} + +async function verifyPublishedIntegrity({ + packageName, + version, + localIntegrity, + registry, + runner, + sleep, + retryDelaysMs, +}) { + let lastResult; + for (const delay of retryDelaysMs) { + if (delay > 0) { + await sleep(delay); + } + lastResult = await runner("npm", [ + "view", + `${packageName}@${version}`, + "dist.integrity", + "--json", + "--registry", + registry, + ]); + if (lastResult.status === 0) { + const registryIntegrity = parseIntegrity(lastResult.stdout); + if (registryIntegrity === localIntegrity) { + return; + } + if (registryIntegrity && registryIntegrity !== localIntegrity) { + throw new Error( + `Published integrity mismatch for ${packageName}@${version}: expected ${localIntegrity}, got ${registryIntegrity}.` + ); + } + } + } + + const output = lastResult ? `${lastResult.stdout}\n${lastResult.stderr}`.trim() : ""; + throw new Error( + `Could not verify published integrity for ${packageName}@${version} after ${retryDelaysMs.length} attempts.${output ? `\n${output}` : ""}` + ); +} + +export async function publishTarball({ + tarball, + registry, + tag, + access, + azure = false, + runner = runCommand, + sleep = (delay) => new Promise((resolve) => setTimeout(resolve, delay)), + retryDelaysMs = DEFAULT_RETRY_DELAYS_MS, + inspect = inspectTarball, +}) { + const packageInfo = await inspect(tarball, runner); + const args = ["publish", tarball, "--tag", tag, "--registry", registry]; + if (access) { + args.push("--access", access); + } + + const result = await runner("npm", args, { stream: true }); + if (result.status === 0) { + return { recoveredConflict: false }; + } + + const output = `${result.stdout}\n${result.stderr}`; + const recognizedConflict = + PUBLIC_CONFLICT.test(output) || (azure && AZURE_TARBALL_CONFLICT.test(output)); + if (!recognizedConflict) { + throw new Error(`npm publish failed with exit code ${result.status}.`); + } + + await verifyPublishedIntegrity({ + packageName: packageInfo.name, + version: packageInfo.version, + localIntegrity: packageInfo.integrity, + registry, + runner, + sleep, + retryDelaysMs, + }); + console.log( + `${packageInfo.name}@${packageInfo.version} already exists with matching integrity; treating the publish conflict as success.` + ); + return { recoveredConflict: true }; +} + +function parseOptions(args) { + const options = {}; + for (let index = 0; index < args.length; index += 2) { + const key = args[index]; + const value = args[index + 1]; + if (!key?.startsWith("--") || value === undefined) { + throw new Error(`Invalid argument: ${key ?? ""}`); + } + options[key.slice(2)] = value; + } + return options; +} + +async function main() { + const [command, ...args] = process.argv.slice(2); + const options = parseOptions(args); + + if (command === "preflight") { + await assertVersionAbsent({ + packageName: options.package, + version: options.version, + registry: options.registry, + }); + console.log(`${options.package}@${options.version} is available on public npm.`); + return; + } + + if (command === "publish") { + await publishTarball({ + tarball: options.tarball, + registry: options.registry, + tag: options.tag, + access: options.access, + azure: options.azure === "true", + }); + return; + } + + throw new Error(`Unknown command: ${command ?? ""}`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(`::error::${error.message}`); + process.exitCode = 1; + }); +} diff --git a/nodejs/test/npm-release.test.ts b/nodejs/test/npm-release.test.ts new file mode 100644 index 0000000000..2e216b2c13 --- /dev/null +++ b/nodejs/test/npm-release.test.ts @@ -0,0 +1,246 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { assertVersionAbsent, publishTarball } from "../scripts/npm-release.js"; + +function integrity(value: string) { + return `sha512-${createHash("sha512").update(value).digest("base64")}`; +} + +const packageInfo = { + name: "@github/copilot-sdk", + version: "1.2.3", + integrity: integrity("match"), +}; + +function result(status: number, stdout = "", stderr = "") { + return { status, stdout, stderr }; +} + +describe("npm release preflight", () => { + it("succeeds only for a definitive not-found response", async () => { + const runner = vi.fn().mockResolvedValue( + result( + 1, + JSON.stringify({ + error: { code: "E404", summary: "No match found for version 1.2.3" }, + }), + "npm error code E404\nnpm error 404" + ) + ); + await expect( + assertVersionAbsent({ + packageName: packageInfo.name, + version: packageInfo.version, + registry: "https://registry.npmjs.org", + runner, + }) + ).resolves.toBeUndefined(); + }); + + it("fails when the version already exists", async () => { + const runner = vi.fn().mockResolvedValue(result(0, JSON.stringify(packageInfo.version))); + await expect( + assertVersionAbsent({ + packageName: packageInfo.name, + version: packageInfo.version, + registry: "https://registry.npmjs.org", + runner, + }) + ).rejects.toThrow("already exists"); + }); + + it("fails on transient registry errors", async () => { + const runner = vi.fn().mockResolvedValue(result(1, "", "npm error code E500")); + await expect( + assertVersionAbsent({ + packageName: packageInfo.name, + version: packageInfo.version, + registry: "https://registry.npmjs.org", + runner, + }) + ).rejects.toThrow("Could not confirm"); + }); + + it("fails when a non-404 error contains E404 and 404 text", async () => { + const runner = vi.fn().mockResolvedValue( + result( + 1, + JSON.stringify({ + error: { + code: "E500", + summary: "Failed to query version 1.2.3-E404.404", + }, + }), + "npm error code E500 for version 1.2.3-E404.404" + ) + ); + await expect( + assertVersionAbsent({ + packageName: packageInfo.name, + version: "1.2.3-E404.404", + registry: "https://registry.npmjs.org", + runner, + }) + ).rejects.toThrow("Could not confirm"); + }); + + it("fails on malformed registry metadata", async () => { + const runner = vi.fn().mockResolvedValue(result(0, "not-json")); + await expect( + assertVersionAbsent({ + packageName: packageInfo.name, + version: packageInfo.version, + registry: "https://registry.npmjs.org", + runner, + }) + ).rejects.toThrow("malformed version metadata"); + }); +}); + +describe("npm release publishing", () => { + const baseOptions = { + tarball: "package.tgz", + registry: "https://registry.example.test", + tag: "latest", + inspect: vi.fn().mockResolvedValue(packageInfo), + sleep: vi.fn().mockResolvedValue(undefined), + retryDelaysMs: [0, 1], + }; + + it("succeeds after a normal publish", async () => { + const runner = vi.fn().mockResolvedValue(result(0)); + await expect(publishTarball({ ...baseOptions, runner })).resolves.toEqual({ + recoveredConflict: false, + }); + }); + + it("recovers a public conflict only when integrity matches", async () => { + const runner = vi + .fn() + .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) + .mockResolvedValueOnce(result(0, JSON.stringify(packageInfo.integrity))); + await expect(publishTarball({ ...baseOptions, runner })).resolves.toEqual({ + recoveredConflict: true, + }); + }); + + it("recovers the known public immutable-version diagnostic", async () => { + const runner = vi + .fn() + .mockResolvedValueOnce( + result( + 1, + "", + "npm error 403 403 Forbidden - PUT https://registry.npmjs.org/@github%2fcopilot-sdk - You cannot publish over the previously published versions: 1.2.3." + ) + ) + .mockResolvedValueOnce(result(0, JSON.stringify(packageInfo.integrity))); + await expect(publishTarball({ ...baseOptions, runner })).resolves.toEqual({ + recoveredConflict: true, + }); + }); + + it("recovers an Azure tarball conflict only when integrity matches", async () => { + const runner = vi + .fn() + .mockResolvedValueOnce( + result( + 1, + "", + "npm error 403 403 Forbidden - PUT https://pkgs.dev.azure.com/feed - already contains file 'github-copilot-sdk-1.2.3.tgz' in package '@github/copilot-sdk/1.2.3'" + ) + ) + .mockResolvedValueOnce(result(0, JSON.stringify(packageInfo.integrity))); + await expect(publishTarball({ ...baseOptions, azure: true, runner })).resolves.toEqual({ + recoveredConflict: true, + }); + }); + + it("fails when published integrity differs", async () => { + const runner = vi + .fn() + .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) + .mockResolvedValueOnce(result(0, JSON.stringify(integrity("different")))); + await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( + "Published integrity mismatch" + ); + }); + + it("fails after bounded retries when integrity is unavailable", async () => { + const runner = vi + .fn() + .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) + .mockResolvedValue(result(1, "", "npm error code E404")); + await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( + "after 2 attempts" + ); + expect(runner).toHaveBeenCalledTimes(3); + }); + + it("fails after bounded retries when integrity is malformed", async () => { + const runner = vi + .fn() + .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) + .mockResolvedValue(result(0, JSON.stringify("sha512-c2hvcnQ="))); + await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( + "after 2 attempts" + ); + expect(runner).toHaveBeenCalledTimes(3); + }); + + it("does not recover a generic Azure 403", async () => { + const runner = vi.fn().mockResolvedValue(result(1, "", "403 Forbidden")); + await expect(publishTarball({ ...baseOptions, azure: true, runner })).rejects.toThrow( + "npm publish failed" + ); + expect(runner).toHaveBeenCalledTimes(1); + }); + + it("does not recover an Azure non-tarball conflict", async () => { + const runner = vi + .fn() + .mockResolvedValue( + result( + 1, + "", + "npm error 403 already contains file 'package.json' in package '@github/copilot-sdk/1.2.3'" + ) + ); + await expect(publishTarball({ ...baseOptions, azure: true, runner })).rejects.toThrow( + "npm publish failed" + ); + expect(runner).toHaveBeenCalledTimes(1); + }); + + it("does not recover an unrelated public error embedding the known phrase", async () => { + const runner = vi + .fn() + .mockResolvedValue( + result( + 1, + "", + "npm error network timeout while parsing 'cannot publish over the previously published versions'" + ) + ); + await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( + "npm publish failed" + ); + expect(runner).toHaveBeenCalledTimes(1); + }); + + it("does not recover an unrelated Azure error embedding the known phrase", async () => { + const runner = vi + .fn() + .mockResolvedValue( + result( + 1, + "", + "npm error network timeout while parsing \"already contains file 'package.tgz' in package '@github/copilot-sdk/1.2.3'\"" + ) + ); + await expect(publishTarball({ ...baseOptions, azure: true, runner })).rejects.toThrow( + "npm publish failed" + ); + expect(runner).toHaveBeenCalledTimes(1); + }); +}); From 5e72985bfe57de0286ce7c4d7aa65574a95b9091 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Fri, 17 Jul 2026 11:50:58 -0700 Subject: [PATCH 2/4] Simplify release retry handling Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b6e76edd-7acc-4681-a884-56d16967c048 --- .github/workflows/publish.yml | 22 +-- nodejs/scripts/npm-release.js | 214 ++++-------------------- nodejs/test/npm-release.test.ts | 286 +++++++------------------------- 3 files changed, 105 insertions(+), 417 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e6eb820c8b..b44fd582a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -81,9 +81,9 @@ jobs: VERSION: ${{ steps.version.outputs.VERSION }} run: | node scripts/npm-release.js preflight \ - --package @github/copilot-sdk \ - --version "$VERSION" \ - --registry https://registry.npmjs.org + @github/copilot-sdk \ + "$VERSION" \ + https://registry.npmjs.org package-nodejs: name: Package Node.js SDK @@ -155,10 +155,10 @@ jobs: exit 1 fi node nodejs/scripts/npm-release.js publish \ - --tarball "${TARBALLS[0]}" \ - --tag "$DIST_TAG" \ - --access public \ - --registry https://registry.npmjs.org + "${TARBALLS[0]}" \ + "$DIST_TAG" \ + https://registry.npmjs.org \ + public publish-nodejs-internal: name: Publish Node.js SDK to internal feed @@ -214,10 +214,10 @@ jobs: exit 1 fi node nodejs/scripts/npm-release.js publish \ - --tarball "${TARBALLS[0]}" \ - --tag "$DIST_TAG" \ - --registry "$FEED_URL" \ - --azure true + "${TARBALLS[0]}" \ + "$DIST_TAG" \ + "$FEED_URL" \ + azure publish-dotnet: name: Publish .NET SDK diff --git a/nodejs/scripts/npm-release.js b/nodejs/scripts/npm-release.js index 73c3967469..5098b5905a 100644 --- a/nodejs/scripts/npm-release.js +++ b/nodejs/scripts/npm-release.js @@ -1,42 +1,33 @@ #!/usr/bin/env node -import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; import { spawn } from "node:child_process"; import { pathToFileURL } from "node:url"; const PUBLIC_CONFLICT = /^(?:npm (?:error|ERR!) code EPUBLISHCONFLICT|npm (?:error|ERR!) (?:403 [^\r\n]* - )?(?:You )?cannot publish over (?:the )?previously published versions(?:: [^\r\n]+)?\.?)\r?$/im; -const AZURE_TARBALL_CONFLICT = +const AZURE_CONFLICT = /^npm (?:error|ERR!) (?:403 [^\r\n]* - )?already contains file '[^'\r\n]+\.tgz' in package '[^'\r\n]+'\.?\r?$/im; -const DEFAULT_RETRY_DELAYS_MS = [0, 1000, 2000, 4000, 8000]; -export async function runCommand(command, args, { stream = false } = {}) { - return await new Promise((resolve, reject) => { +export function runCommand(command, args, { stream = false } = {}) { + return new Promise((resolve, reject) => { const child = spawn(command, args, { shell: false }); let stdout = ""; let stderr = ""; child.stdout.on("data", (chunk) => { - const text = chunk.toString(); - stdout += text; - if (stream) { - process.stdout.write(chunk); - } + stdout += chunk; + if (stream) process.stdout.write(chunk); }); child.stderr.on("data", (chunk) => { - const text = chunk.toString(); - stderr += text; - if (stream) { - process.stderr.write(chunk); - } + stderr += chunk; + if (stream) process.stderr.write(chunk); }); child.on("error", reject); child.on("close", (status) => resolve({ status: status ?? 1, stdout, stderr })); }); } -export async function assertVersionAbsent({ packageName, version, registry, runner = runCommand }) { +export async function assertVersionAbsent(packageName, version, registry, runner = runCommand) { const result = await runner("npm", [ "view", `${packageName}@${version}`, @@ -45,199 +36,54 @@ export async function assertVersionAbsent({ packageName, version, registry, runn "--registry", registry, ]); - const output = `${result.stdout}\n${result.stderr}`; if (result.status === 0) { - let publishedVersion; - try { - publishedVersion = JSON.parse(result.stdout); - } catch { - throw new Error( - `Public npm returned malformed version metadata for ${packageName}@${version}.` - ); - } - if (publishedVersion !== version) { - throw new Error( - `Public npm returned unexpected version metadata for ${packageName}@${version}: ${result.stdout.trim()}` - ); - } throw new Error(`${packageName}@${version} already exists on public npm.`); } try { - const errorResult = JSON.parse(result.stdout); - if (errorResult?.error?.code === "E404") { - return; - } + if (JSON.parse(result.stdout)?.error?.code === "E404") return; } catch { // The failure below includes npm's output for diagnosis. } + const output = `${result.stdout}\n${result.stderr}`.trim(); throw new Error( - `Could not confirm that ${packageName}@${version} is absent from public npm (npm exited ${result.status}).\n${output.trim()}` - ); -} - -export async function inspectTarball(tarball, runner = runCommand) { - const result = await runner("tar", ["-xOf", tarball, "package/package.json"]); - if (result.status !== 0) { - throw new Error( - `Could not read package/package.json from ${tarball}.\n${result.stderr.trim()}` - ); - } - - let manifest; - try { - manifest = JSON.parse(result.stdout); - } catch { - throw new Error(`Tarball ${tarball} contains malformed package metadata.`); - } - if (typeof manifest.name !== "string" || typeof manifest.version !== "string") { - throw new Error(`Tarball ${tarball} does not contain a valid package name and version.`); - } - - const bytes = await readFile(tarball); - const integrity = `sha512-${createHash("sha512").update(bytes).digest("base64")}`; - return { name: manifest.name, version: manifest.version, integrity }; -} - -function parseIntegrity(stdout) { - try { - const integrity = JSON.parse(stdout); - if (typeof integrity !== "string") { - return null; - } - const match = /^sha512-([A-Za-z0-9+/]+={0,2})$/.exec(integrity); - return match && Buffer.from(match[1], "base64").length === 64 ? integrity : null; - } catch { - return null; - } -} - -async function verifyPublishedIntegrity({ - packageName, - version, - localIntegrity, - registry, - runner, - sleep, - retryDelaysMs, -}) { - let lastResult; - for (const delay of retryDelaysMs) { - if (delay > 0) { - await sleep(delay); - } - lastResult = await runner("npm", [ - "view", - `${packageName}@${version}`, - "dist.integrity", - "--json", - "--registry", - registry, - ]); - if (lastResult.status === 0) { - const registryIntegrity = parseIntegrity(lastResult.stdout); - if (registryIntegrity === localIntegrity) { - return; - } - if (registryIntegrity && registryIntegrity !== localIntegrity) { - throw new Error( - `Published integrity mismatch for ${packageName}@${version}: expected ${localIntegrity}, got ${registryIntegrity}.` - ); - } - } - } - - const output = lastResult ? `${lastResult.stdout}\n${lastResult.stderr}`.trim() : ""; - throw new Error( - `Could not verify published integrity for ${packageName}@${version} after ${retryDelaysMs.length} attempts.${output ? `\n${output}` : ""}` + `Could not confirm that ${packageName}@${version} is absent from public npm (npm exited ${result.status}).${output ? `\n${output}` : ""}` ); } -export async function publishTarball({ - tarball, - registry, - tag, - access, - azure = false, - runner = runCommand, - sleep = (delay) => new Promise((resolve) => setTimeout(resolve, delay)), - retryDelaysMs = DEFAULT_RETRY_DELAYS_MS, - inspect = inspectTarball, -}) { - const packageInfo = await inspect(tarball, runner); +export async function publishTarball(tarball, tag, registry, mode, runner = runCommand) { const args = ["publish", tarball, "--tag", tag, "--registry", registry]; - if (access) { - args.push("--access", access); - } + if (mode === "public") args.push("--access", "public"); + if (mode !== "public" && mode !== "azure") throw new Error(`Unknown publish mode: ${mode}`); const result = await runner("npm", args, { stream: true }); - if (result.status === 0) { - return { recoveredConflict: false }; - } + if (result.status === 0) return; const output = `${result.stdout}\n${result.stderr}`; - const recognizedConflict = - PUBLIC_CONFLICT.test(output) || (azure && AZURE_TARBALL_CONFLICT.test(output)); - if (!recognizedConflict) { - throw new Error(`npm publish failed with exit code ${result.status}.`); + if (PUBLIC_CONFLICT.test(output) || (mode === "azure" && AZURE_CONFLICT.test(output))) { + console.log( + "Version already published; treating the immutable-version conflict as success." + ); + return; } - await verifyPublishedIntegrity({ - packageName: packageInfo.name, - version: packageInfo.version, - localIntegrity: packageInfo.integrity, - registry, - runner, - sleep, - retryDelaysMs, - }); - console.log( - `${packageInfo.name}@${packageInfo.version} already exists with matching integrity; treating the publish conflict as success.` - ); - return { recoveredConflict: true }; -} - -function parseOptions(args) { - const options = {}; - for (let index = 0; index < args.length; index += 2) { - const key = args[index]; - const value = args[index + 1]; - if (!key?.startsWith("--") || value === undefined) { - throw new Error(`Invalid argument: ${key ?? ""}`); - } - options[key.slice(2)] = value; - } - return options; + throw new Error(`npm publish failed with exit code ${result.status}.`); } async function main() { const [command, ...args] = process.argv.slice(2); - const options = parseOptions(args); - - if (command === "preflight") { - await assertVersionAbsent({ - packageName: options.package, - version: options.version, - registry: options.registry, - }); - console.log(`${options.package}@${options.version} is available on public npm.`); - return; - } - - if (command === "publish") { - await publishTarball({ - tarball: options.tarball, - registry: options.registry, - tag: options.tag, - access: options.access, - azure: options.azure === "true", - }); - return; + if (command === "preflight" && args.length === 3) { + await assertVersionAbsent(...args); + console.log(`${args[0]}@${args[1]} is available on public npm.`); + } else if (command === "publish" && args.length === 4) { + await publishTarball(...args); + } else { + throw new Error( + "Usage: npm-release.js preflight | publish " + ); } - - throw new Error(`Unknown command: ${command ?? ""}`); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/nodejs/test/npm-release.test.ts b/nodejs/test/npm-release.test.ts index 2e216b2c13..028df99d4e 100644 --- a/nodejs/test/npm-release.test.ts +++ b/nodejs/test/npm-release.test.ts @@ -1,246 +1,88 @@ -import { createHash } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; import { assertVersionAbsent, publishTarball } from "../scripts/npm-release.js"; -function integrity(value: string) { - return `sha512-${createHash("sha512").update(value).digest("base64")}`; -} - -const packageInfo = { - name: "@github/copilot-sdk", - version: "1.2.3", - integrity: integrity("match"), -}; - -function result(status: number, stdout = "", stderr = "") { - return { status, stdout, stderr }; -} +const packageName = "@github/copilot-sdk"; +const version = "1.2.3"; +const registry = "https://registry.example.test"; +const result = (status: number, stdout = "", stderr = "") => ({ status, stdout, stderr }); describe("npm release preflight", () => { - it("succeeds only for a definitive not-found response", async () => { - const runner = vi.fn().mockResolvedValue( - result( - 1, - JSON.stringify({ - error: { code: "E404", summary: "No match found for version 1.2.3" }, - }), - "npm error code E404\nnpm error 404" - ) - ); + it("succeeds only for a structured E404 response", async () => { + const runner = vi + .fn() + .mockResolvedValue(result(1, JSON.stringify({ error: { code: "E404" } }))); await expect( - assertVersionAbsent({ - packageName: packageInfo.name, - version: packageInfo.version, - registry: "https://registry.npmjs.org", - runner, - }) + assertVersionAbsent(packageName, version, registry, runner) ).resolves.toBeUndefined(); }); - it("fails when the version already exists", async () => { - const runner = vi.fn().mockResolvedValue(result(0, JSON.stringify(packageInfo.version))); - await expect( - assertVersionAbsent({ - packageName: packageInfo.name, - version: packageInfo.version, - registry: "https://registry.npmjs.org", - runner, - }) - ).rejects.toThrow("already exists"); - }); - - it("fails on transient registry errors", async () => { - const runner = vi.fn().mockResolvedValue(result(1, "", "npm error code E500")); - await expect( - assertVersionAbsent({ - packageName: packageInfo.name, - version: packageInfo.version, - registry: "https://registry.npmjs.org", - runner, - }) - ).rejects.toThrow("Could not confirm"); - }); - - it("fails when a non-404 error contains E404 and 404 text", async () => { - const runner = vi.fn().mockResolvedValue( + it.each([ + ["an existing version", result(0, JSON.stringify(version)), "already exists"], + ["a transient error", result(1, "", "npm error code E500"), "Could not confirm"], + ["malformed output", result(1, "not-json"), "Could not confirm"], + [ + "a non-404 error containing E404 and 404 text", result( 1, - JSON.stringify({ - error: { - code: "E500", - summary: "Failed to query version 1.2.3-E404.404", - }, - }), - "npm error code E500 for version 1.2.3-E404.404" - ) + JSON.stringify({ error: { code: "E500", summary: "version 1.2.3-E404.404" } }), + "npm error code E500 for 1.2.3-E404.404" + ), + "Could not confirm", + ], + ])("fails for %s", async (_name, response, message) => { + const runner = vi.fn().mockResolvedValue(response); + await expect(assertVersionAbsent(packageName, version, registry, runner)).rejects.toThrow( + message ); - await expect( - assertVersionAbsent({ - packageName: packageInfo.name, - version: "1.2.3-E404.404", - registry: "https://registry.npmjs.org", - runner, - }) - ).rejects.toThrow("Could not confirm"); - }); - - it("fails on malformed registry metadata", async () => { - const runner = vi.fn().mockResolvedValue(result(0, "not-json")); - await expect( - assertVersionAbsent({ - packageName: packageInfo.name, - version: packageInfo.version, - registry: "https://registry.npmjs.org", - runner, - }) - ).rejects.toThrow("malformed version metadata"); }); }); describe("npm release publishing", () => { - const baseOptions = { - tarball: "package.tgz", - registry: "https://registry.example.test", - tag: "latest", - inspect: vi.fn().mockResolvedValue(packageInfo), - sleep: vi.fn().mockResolvedValue(undefined), - retryDelaysMs: [0, 1], - }; - it("succeeds after a normal publish", async () => { const runner = vi.fn().mockResolvedValue(result(0)); - await expect(publishTarball({ ...baseOptions, runner })).resolves.toEqual({ - recoveredConflict: false, - }); - }); - - it("recovers a public conflict only when integrity matches", async () => { - const runner = vi - .fn() - .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) - .mockResolvedValueOnce(result(0, JSON.stringify(packageInfo.integrity))); - await expect(publishTarball({ ...baseOptions, runner })).resolves.toEqual({ - recoveredConflict: true, - }); - }); - - it("recovers the known public immutable-version diagnostic", async () => { - const runner = vi - .fn() - .mockResolvedValueOnce( - result( - 1, - "", - "npm error 403 403 Forbidden - PUT https://registry.npmjs.org/@github%2fcopilot-sdk - You cannot publish over the previously published versions: 1.2.3." - ) - ) - .mockResolvedValueOnce(result(0, JSON.stringify(packageInfo.integrity))); - await expect(publishTarball({ ...baseOptions, runner })).resolves.toEqual({ - recoveredConflict: true, - }); - }); - - it("recovers an Azure tarball conflict only when integrity matches", async () => { - const runner = vi - .fn() - .mockResolvedValueOnce( - result( - 1, - "", - "npm error 403 403 Forbidden - PUT https://pkgs.dev.azure.com/feed - already contains file 'github-copilot-sdk-1.2.3.tgz' in package '@github/copilot-sdk/1.2.3'" - ) - ) - .mockResolvedValueOnce(result(0, JSON.stringify(packageInfo.integrity))); - await expect(publishTarball({ ...baseOptions, azure: true, runner })).resolves.toEqual({ - recoveredConflict: true, - }); - }); - - it("fails when published integrity differs", async () => { - const runner = vi - .fn() - .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) - .mockResolvedValueOnce(result(0, JSON.stringify(integrity("different")))); - await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( - "Published integrity mismatch" - ); - }); - - it("fails after bounded retries when integrity is unavailable", async () => { - const runner = vi - .fn() - .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) - .mockResolvedValue(result(1, "", "npm error code E404")); - await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( - "after 2 attempts" - ); - expect(runner).toHaveBeenCalledTimes(3); - }); - - it("fails after bounded retries when integrity is malformed", async () => { - const runner = vi - .fn() - .mockResolvedValueOnce(result(1, "", "npm error code EPUBLISHCONFLICT")) - .mockResolvedValue(result(0, JSON.stringify("sha512-c2hvcnQ="))); - await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( - "after 2 attempts" - ); - expect(runner).toHaveBeenCalledTimes(3); - }); - - it("does not recover a generic Azure 403", async () => { - const runner = vi.fn().mockResolvedValue(result(1, "", "403 Forbidden")); - await expect(publishTarball({ ...baseOptions, azure: true, runner })).rejects.toThrow( - "npm publish failed" - ); - expect(runner).toHaveBeenCalledTimes(1); - }); - - it("does not recover an Azure non-tarball conflict", async () => { - const runner = vi - .fn() - .mockResolvedValue( - result( - 1, - "", - "npm error 403 already contains file 'package.json' in package '@github/copilot-sdk/1.2.3'" - ) - ); - await expect(publishTarball({ ...baseOptions, azure: true, runner })).rejects.toThrow( - "npm publish failed" - ); - expect(runner).toHaveBeenCalledTimes(1); + await expect( + publishTarball("package.tgz", "latest", registry, "public", runner) + ).resolves.toBeUndefined(); }); - it("does not recover an unrelated public error embedding the known phrase", async () => { - const runner = vi - .fn() - .mockResolvedValue( - result( - 1, - "", - "npm error network timeout while parsing 'cannot publish over the previously published versions'" - ) - ); - await expect(publishTarball({ ...baseOptions, runner })).rejects.toThrow( - "npm publish failed" - ); - expect(runner).toHaveBeenCalledTimes(1); + it.each([ + ["npm error code EPUBLISHCONFLICT", "public"], + [ + "npm error 403 403 Forbidden - PUT https://registry.npmjs.org/package - You cannot publish over the previously published versions: 1.2.3.", + "public", + ], + [ + "npm error 403 403 Forbidden - PUT https://pkgs.dev.azure.com/feed - already contains file 'package.tgz' in package '@github/copilot-sdk/1.2.3'", + "azure", + ], + ])("recovers the immutable conflict: %s", async (error, mode) => { + const runner = vi.fn().mockResolvedValue(result(1, "", error)); + await expect( + publishTarball("package.tgz", "latest", registry, mode, runner) + ).resolves.toBeUndefined(); }); - it("does not recover an unrelated Azure error embedding the known phrase", async () => { - const runner = vi - .fn() - .mockResolvedValue( - result( - 1, - "", - "npm error network timeout while parsing \"already contains file 'package.tgz' in package '@github/copilot-sdk/1.2.3'\"" - ) - ); - await expect(publishTarball({ ...baseOptions, azure: true, runner })).rejects.toThrow( - "npm publish failed" - ); - expect(runner).toHaveBeenCalledTimes(1); + it.each([ + ["a generic Azure 403", "403 Forbidden", "azure"], + [ + "an Azure non-tarball conflict", + "npm error 403 already contains file 'package.json' in package '@github/copilot-sdk/1.2.3'", + "azure", + ], + [ + "an embedded public phrase", + "npm error network timeout while parsing 'cannot publish over the previously published versions'", + "public", + ], + [ + "an embedded Azure phrase", + "npm error network timeout while parsing \"already contains file 'package.tgz' in package '@github/copilot-sdk/1.2.3'\"", + "azure", + ], + ])("fails for %s", async (_name, error, mode) => { + const runner = vi.fn().mockResolvedValue(result(1, "", error)); + await expect( + publishTarball("package.tgz", "latest", registry, mode, runner) + ).rejects.toThrow("npm publish failed"); }); }); From ed64363ee17e3639e1ef79d88e7e268c826e00bc Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Fri, 17 Jul 2026 14:18:13 -0700 Subject: [PATCH 3/4] Handle Azure feed conflict prefix Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b6e76edd-7acc-4681-a884-56d16967c048 --- nodejs/scripts/npm-release.js | 2 +- nodejs/test/npm-release.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nodejs/scripts/npm-release.js b/nodejs/scripts/npm-release.js index 5098b5905a..c2cdb9bc80 100644 --- a/nodejs/scripts/npm-release.js +++ b/nodejs/scripts/npm-release.js @@ -6,7 +6,7 @@ import { pathToFileURL } from "node:url"; const PUBLIC_CONFLICT = /^(?:npm (?:error|ERR!) code EPUBLISHCONFLICT|npm (?:error|ERR!) (?:403 [^\r\n]* - )?(?:You )?cannot publish over (?:the )?previously published versions(?:: [^\r\n]+)?\.?)\r?$/im; const AZURE_CONFLICT = - /^npm (?:error|ERR!) (?:403 [^\r\n]* - )?already contains file '[^'\r\n]+\.tgz' in package '[^'\r\n]+'\.?\r?$/im; + /^npm (?:error|ERR!) (?:403 [^\r\n]* - )?(?:The feed '[^'\r\n]+' )?already contains file '[^'\r\n]+\.tgz' in package '[^'\r\n]+'\.?\r?$/im; export function runCommand(command, args, { stream = false } = {}) { return new Promise((resolve, reject) => { diff --git a/nodejs/test/npm-release.test.ts b/nodejs/test/npm-release.test.ts index 028df99d4e..26caf7deaa 100644 --- a/nodejs/test/npm-release.test.ts +++ b/nodejs/test/npm-release.test.ts @@ -52,7 +52,7 @@ describe("npm release publishing", () => { "public", ], [ - "npm error 403 403 Forbidden - PUT https://pkgs.dev.azure.com/feed - already contains file 'package.tgz' in package '@github/copilot-sdk/1.2.3'", + "npm error 403 403 Forbidden - The feed 'copilot-canary' already contains file 'copilot-sdk-0.0.0-29613896246.tgz' in package '@github/copilot-sdk 0.0.0-29613896246'.", "azure", ], ])("recovers the immutable conflict: %s", async (error, mode) => { From db07b55d4a5bb0ddd67d3f2a9be0c265dab1e5dc Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Fri, 17 Jul 2026 14:52:29 -0700 Subject: [PATCH 4/4] Fix Windows release helper parsing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b6e76edd-7acc-4681-a884-56d16967c048 --- nodejs/scripts/npm-release.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/nodejs/scripts/npm-release.js b/nodejs/scripts/npm-release.js index c2cdb9bc80..fe750bada0 100644 --- a/nodejs/scripts/npm-release.js +++ b/nodejs/scripts/npm-release.js @@ -1,5 +1,3 @@ -#!/usr/bin/env node - import { spawn } from "node:child_process"; import { pathToFileURL } from "node:url";