diff --git a/.github/workflows/spec-sync.yml b/.github/workflows/spec-sync.yml index 28f38f40..0f68d192 100644 --- a/.github/workflows/spec-sync.yml +++ b/.github/workflows/spec-sync.yml @@ -32,9 +32,15 @@ jobs: specs: ${{ steps.list.outputs.specs }} count: ${{ steps.list.outputs.count }} errors: ${{ steps.list.outputs.errors }} + stale: ${{ steps.list.outputs.stale }} + staleCount: ${{ steps.list.outputs.staleCount }} steps: - uses: actions/checkout@v4 + # The comparison is against origin/main, which a shallow checkout of another ref lacks. + - name: Fetch the baseline ref + run: git fetch --no-tags --depth=1 origin main:refs/remotes/origin/main + - uses: actions/setup-node@v3 with: node-version: 22.x @@ -54,6 +60,11 @@ jobs: if [ "${{ steps.list.outputs.errors }}" != "0" ]; then echo "${{ steps.list.outputs.errors }} spec(s) could not be read from the docs site." >> "$GITHUB_STEP_SUMMARY" fi + # A spec published far ahead of main has stopped reaching main, whatever the reason. + if [ "${{ steps.list.outputs.staleCount }}" != "0" ]; then + echo "::warning::${{ steps.list.outputs.staleCount }} spec(s) behind the published spec for over 14 days: ${{ steps.list.outputs.stale }}" + echo "Behind the published spec for over 14 days: ${{ steps.list.outputs.stale }}" >> "$GITHUB_STEP_SUMMARY" + fi sync: name: "Sync ${{ matrix.spec }}" diff --git a/scripts/spec-sync/README.md b/scripts/spec-sync/README.md index f3a46815..a590cbe4 100644 --- a/scripts/spec-sync/README.md +++ b/scripts/spec-sync/README.md @@ -19,8 +19,18 @@ node scripts/spec-sync/sync-spec.mjs --spec pim # refresh, regenerate, ``` `--dry-run` stops `sync-spec` before it touches the working spec. `SPEC_SYNC_BASELINE` -overrides the ref the export diff compares against (default `origin/main`). -`SPEC_SYNC_BASE_URL` overrides where specs are downloaded from. +overrides the ref both the "does this spec need refreshing?" comparison and the export diff use +(default `origin/main`); it must resolve, or both scripts stop with an error. +`SPEC_SYNC_BASE_URL` overrides where specs are downloaded from. `SPEC_SYNC_STALE_DAYS` +(default 14) sets how far a published spec may run ahead of the baseline before `--list` warns. + +## The baseline + +Both scripts compare the published spec against `packages/sdks/specs/.yaml` **on the +baseline ref**, never against the working tree. The sync job checks out the spec's long-lived +`spec-sync/` branch, so a working-tree comparison would ask whether that branch is up to +date rather than whether `main` is — and a branch outliving a closed pull request would report +its spec as current forever. A spec absent from the baseline counts as fully changed. ## What it does diff --git a/scripts/spec-sync/baseline.mjs b/scripts/spec-sync/baseline.mjs new file mode 100644 index 00000000..b93452ad --- /dev/null +++ b/scripts/spec-sync/baseline.mjs @@ -0,0 +1,60 @@ +/** + * The ref spec-sync compares published specs against, and the helpers that read it. + * + * The comparison asks whether the published spec differs from what is released on the baseline, + * not from whatever the checked-out branch happens to hold. + */ +import { execFileSync } from "node:child_process" +import { resolve, dirname } from "node:path" +import { fileURLToPath } from "node:url" + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..") + +export const BASELINE = process.env.SPEC_SYNC_BASELINE ?? "origin/main" + +const git = (args) => + execFileSync("git", args, { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 64 * 1024 * 1024, + }) + +let checked = false + +/** An unresolvable ref would silently report every spec as changed, so it is an error instead. */ +export function assertBaselineResolves() { + if (checked) return + try { + git(["rev-parse", "--verify", "--quiet", `${BASELINE}^{commit}`]) + } catch { + throw new Error( + `baseline ref "${BASELINE}" cannot be resolved. Fetch it (git fetch origin main), ` + + `or point SPEC_SYNC_BASELINE at a ref that exists.`, + ) + } + checked = true +} + +/** The spec as it stands on the baseline ref, or "" when the ref does not carry that file yet. */ +export function specOnBaseline(specFile) { + assertBaselineResolves() + try { + return git(["show", `${BASELINE}:packages/sdks/specs/${specFile}`]) + } catch { + return "" // a spec added since the baseline, so everything published is new + } +} + +/** UTC day count between two `x-version-timestamp` values, or null when either is unreadable. */ +export function daysBetweenStamps(newer, older) { + const a = Date.parse(newer ?? "") + const b = Date.parse(older ?? "") + if (Number.isNaN(a) || Number.isNaN(b)) return null + return Math.floor((a - b) / 86_400_000) +} + +/** The `info.x-version-timestamp` an OpenAPI document declares, or null. */ +export function versionStamp(spec) { + return /^\s{2}x-version-timestamp:\s*['"]?([^'"\s]+)['"]?\s*$/m.exec(spec)?.[1] ?? null +} diff --git a/scripts/spec-sync/fetch-upstream.mjs b/scripts/spec-sync/fetch-upstream.mjs index e368a749..fe81e89c 100644 --- a/scripts/spec-sync/fetch-upstream.mjs +++ b/scripts/spec-sync/fetch-upstream.mjs @@ -10,14 +10,16 @@ * * Exit codes: 0 = done, 2 = nothing to do, 1 = error. */ -import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs" +import { readFileSync, writeFileSync, mkdirSync } from "node:fs" import { resolve, dirname } from "node:path" import { fileURLToPath } from "node:url" +import { BASELINE, assertBaselineResolves, specOnBaseline, versionStamp, daysBetweenStamps } from "./baseline.mjs" const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..") const specsDir = resolve(repoRoot, "packages/sdks/specs") const config = JSON.parse(readFileSync(resolve(specsDir, "config/canonical-map.json"), "utf8")) const baseUrl = process.env.SPEC_SYNC_BASE_URL ?? "https://developer.elasticpath.com/assets/openapispecs" +const staleDays = Number(process.env.SPEC_SYNC_STALE_DAYS ?? 14) const args = process.argv.slice(2) const listMode = args.includes("--list") @@ -51,6 +53,12 @@ function syncable() { }) } +try { + assertBaselineResolves() +} catch (err) { + fail(err.message) +} + const rows = syncable() if (specArg && rows.length === 0) { const row = config.specs[specArg] @@ -62,9 +70,10 @@ const results = await Promise.all( rows.map(async ([key, row]) => { try { const canonical = await fetchSpec(key, row) - const workingPath = resolve(specsDir, row.spec) - const working = existsSync(workingPath) ? readFileSync(workingPath, "utf8") : "" - return { key, row, canonical, changed: canonical !== working } + // Against the baseline, not the working tree: the sync job runs on the spec's own branch. + const released = specOnBaseline(row.spec) + const behindDays = daysBetweenStamps(versionStamp(canonical), versionStamp(released)) + return { key, row, canonical, changed: canonical !== released, behindDays } } catch (err) { return { key, row, error: err.message } } @@ -76,8 +85,14 @@ for (const r of errors) console.error(`fetch-upstream: ${r.error}`) if (listMode) { const changed = results.filter((r) => !r.error && r.changed).map((r) => r.key) + // A spec whose published stamp has run ahead of the baseline for this long stopped syncing. + const stale = results.filter((r) => !r.error && r.changed && r.behindDays > staleDays) for (const r of results.filter((r) => !r.error)) { - console.error(` ${r.key.padEnd(22)} ${r.changed ? "differs from ours" : "already current"}`) + const age = r.changed && r.behindDays > 0 ? ` (published ${r.behindDays} day(s) ahead of ${BASELINE})` : "" + console.error(` ${r.key.padEnd(22)} ${r.changed ? "differs from ours" : "already current"}${age}`) + } + for (const r of stale) { + console.error(`fetch-upstream: ${r.key} has been behind the published spec for ${r.behindDays} days`) } // A spec the site would not serve must not look like "already current". It is reported as an // error count rather than an exit code, so the specs that did download still sync; the @@ -85,7 +100,8 @@ if (listMode) { if (process.env.GITHUB_OUTPUT) { writeFileSync( process.env.GITHUB_OUTPUT, - `specs=${JSON.stringify(changed)}\ncount=${changed.length}\nerrors=${errors.length}\n`, + `specs=${JSON.stringify(changed)}\ncount=${changed.length}\nerrors=${errors.length}\n` + + `stale=${JSON.stringify(stale.map((r) => `${r.key} (${r.behindDays}d)`))}\nstaleCount=${stale.length}\n`, { flag: "a" }, ) } @@ -98,7 +114,7 @@ if (!specArg) fail("pass --spec , or --list to see what changed") const [result] = results if (result.error) fail(result.error) if (!result.changed) { - console.log(`fetch-upstream: ${result.key} already matches packages/sdks/specs/${result.row.spec}`) + console.log(`fetch-upstream: ${result.key} already matches packages/sdks/specs/${result.row.spec} on ${BASELINE}`) process.exit(2) } diff --git a/scripts/spec-sync/sync-spec.mjs b/scripts/spec-sync/sync-spec.mjs index 3bc1554a..7c88ec4e 100644 --- a/scripts/spec-sync/sync-spec.mjs +++ b/scripts/spec-sync/sync-spec.mjs @@ -16,11 +16,11 @@ import { readFileSync, writeFileSync, existsSync, copyFileSync, readdirSync } fr import { resolve, dirname } from "node:path" import { tmpdir } from "node:os" import { fileURLToPath } from "node:url" +import { BASELINE, assertBaselineResolves, specOnBaseline } from "./baseline.mjs" const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..") const specsDir = resolve(repoRoot, "packages/sdks/specs") const configPath = resolve(specsDir, "config/canonical-map.json") -const BASELINE = process.env.SPEC_SYNC_BASELINE ?? "origin/main" const args = process.argv.slice(2) const specKey = args[args.indexOf("--spec") + 1] @@ -70,9 +70,16 @@ const workingPath = resolve(specsDir, row.spec) if (!existsSync(upstreamPath)) fail(`nothing downloaded to specs/upstream/${row.upstream}`, 2) const upstream = readFileSync(upstreamPath, "utf8") -const working = existsSync(workingPath) ? readFileSync(workingPath, "utf8") : "" -if (upstream === working) { - console.log(`spec-sync: specs/${row.spec} already matches upstream. Nothing to do.`) +try { + assertBaselineResolves() +} catch (err) { + fail(err.message) +} +// Against the baseline, not the working tree: a `spec-sync/` branch already carrying the +// refresh must still be regenerated, so its pull request is updated rather than skipped. +const released = specOnBaseline(row.spec) +if (upstream === released) { + console.log(`spec-sync: specs/${row.spec} on ${BASELINE} already matches upstream. Nothing to do.`) process.exit(2) }