From 985636b554266b25b04998e2f98096e23e431d7b Mon Sep 17 00:00:00 2001 From: Joris Wouter Jonkers Date: Wed, 26 Aug 2026 11:55:56 +0200 Subject: [PATCH] chore(scripts): collect the cutover and recovery helpers into infra/scripts Five operational scripts were sitting untracked in the repository root, where they survived only as long as the working copy. Each one encodes a procedure that is worth having on the next incident: fix-vault-bootstrap-auth.sh creates the vault-bootstrap policy and kubernetes auth role, and proves the ServiceAccount can log in from inside the cluster. Needed again after any Vault re-initialisation. resume-mail-provisioner.sh resumes apps-mail and verifies the provisioning sidecar, re-suspending and rolling back on any failed check. flip-flux-source-to-org.sh points the Flux source at the organisation repository with GitHub App auth. flip-flux-source-to-monorepo.sh the matching rollback, including the spec.provider removal that a plain apply does not undo. post-cutover-fixups.sh writes the Discord webhook and mints the Vault metrics token. Renamed to the kebab-case the directory already uses, and the cross-reference between the two flip scripts follows the rename. flip-flux-source-to-monorepo.sh derived the monorepo path from a hardcoded absolute path under one user's home directory. Now that it lives in the tree it derives the root from its own location instead, verified to resolve correctly when invoked by relative path, by absolute path, and from an unrelated working directory. flip-flux-source-to-org.sh runs under `set -uo pipefail` with no `-e`, so its unguarded `cd "$WORK"` would have continued into the wrong tree had the cd failed. It now dies instead. Not included: tmp.sh contains a live Discord webhook URL and is scratch rather than a procedure; tmp.js, tpm.sh and tmp.sh.bak-qbt are one-off fragments; and unseal-vault.sh was not part of this work. --- infra/scripts/fix-vault-bootstrap-auth.sh | 206 ++++++++++++++++++ infra/scripts/flip-flux-source-to-monorepo.sh | 69 ++++++ infra/scripts/flip-flux-source-to-org.sh | 86 ++++++++ infra/scripts/post-cutover-fixups.sh | 178 +++++++++++++++ infra/scripts/resume-mail-provisioner.sh | 116 ++++++++++ 5 files changed, 655 insertions(+) create mode 100755 infra/scripts/fix-vault-bootstrap-auth.sh create mode 100755 infra/scripts/flip-flux-source-to-monorepo.sh create mode 100755 infra/scripts/flip-flux-source-to-org.sh create mode 100755 infra/scripts/post-cutover-fixups.sh create mode 100755 infra/scripts/resume-mail-provisioner.sh diff --git a/infra/scripts/fix-vault-bootstrap-auth.sh b/infra/scripts/fix-vault-bootstrap-auth.sh new file mode 100755 index 00000000..b225b63f --- /dev/null +++ b/infra/scripts/fix-vault-bootstrap-auth.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# One-time: give vault-bootstrap-auth a Kubernetes auth role so it stops +# depending on a static token that expires unnoticed. +# +# The Job authenticates with a token from the vault-bootstrap-token Secret. That +# token is invalid (`vault token lookup` -> 403), so the Job has been failing, +# its pods are reaped, and every auth role and policy added to bootstrap-auth.sh +# since it expired was applied by Flux and never reached the server. Three +# backups are blocked on exactly that. +# +# This needs a privileged Vault token ONCE. Afterwards the Job logs in with its +# ServiceAccount and there is nothing left to expire. +# +# The companion manifest change is already merged (fleet-infra #128), so the Job +# currently fails with `invalid role name "vault-bootstrap"` -- this script is +# what creates that role. It only adds a policy and a role; it changes nothing +# that is currently working. +set -euo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +NS=data-system +SA=vault-bootstrap-auth +ROLE=vault-bootstrap +POLICY=vault-bootstrap +PORT=18200 +PF_PID="" + +k() { kubectl --context "$CTX" "$@"; } +say() { printf '\n== %s\n' "$*"; } +die() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } +cleanup() { + [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null || true + k delete pod vault-bootstrap-login-probe -n "$NS" --ignore-not-found >/dev/null 2>&1 || true +} +trap cleanup EXIT + +say "preflight" +k config current-context >/dev/null || die "context $CTX unreachable" +printf ' context: %s\n' "$(k config current-context)" +k get sa "$SA" -n "$NS" >/dev/null 2>&1 || die "ServiceAccount $NS/$SA not found" +printf ' serviceaccount: %s/%s\n' "$NS" "$SA" + +# Vault's TokenReview call needs the SA bound to system:auth-delegator. The Job +# manifest already installs that binding; check rather than assume. +if k get clusterrolebinding vault-bootstrap-auth-token-reviewer >/dev/null 2>&1; then + printf ' token-reviewer binding: present\n' +else + printf ' token-reviewer binding: MISSING -- kubernetes auth login will fail\n' >&2 + die "apply cluster/flux/apps/data/vault/bootstrap-auth-job.yaml first" +fi + +say "opening a port-forward to vault" +k port-forward -n "$NS" svc/vault "$PORT":8200 >/dev/null 2>&1 & +PF_PID=$! +export VAULT_ADDR="http://127.0.0.1:${PORT}" +ready=no +for _ in $(seq 1 20); do + if curl -fsS --max-time 2 "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200" >/dev/null 2>&1; then + ready=yes; break + fi + sleep 2 +done +[ "$ready" = yes ] || die "vault is not reachable on $VAULT_ADDR" + +sealed="$(curl -fsS "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200" | sed -n 's/.*"sealed":\([a-z]*\).*/\1/p')" +printf ' sealed: %s\n' "$sealed" +[ "$sealed" = "false" ] || die "vault is sealed -- unseal before bootstrapping auth" + +say "privileged token" +printf 'Paste a Vault token allowed to write policies and auth roles.\n' +printf 'The initial root token works. Input is hidden.\n' +printf 'token: ' +read -rs VAULT_TOKEN +printf '\n' +export VAULT_TOKEN +[ -n "$VAULT_TOKEN" ] || die "no token provided" + +command -v vault >/dev/null 2>&1 \ + || die "the vault CLI is not installed locally; install it and re-run" +vault token lookup >/dev/null 2>&1 || die "that token is not valid against $VAULT_ADDR" +printf ' token accepted\n' + +say "writing policy $POLICY" +# Scoped to what bootstrap-auth.sh actually does: enable the kubernetes auth +# method and the kvv2/database/rabbitmq engines, write the policies and roles it +# defines, manage the auth-api transit signing key, and read the engine admin +# credentials it configures those engines with. +vault policy write "$POLICY" - <<'HCL' +path "sys/auth" { + capabilities = ["read", "list"] +} +path "sys/auth/*" { + capabilities = ["create", "update", "read", "sudo"] +} +path "sys/mounts" { + capabilities = ["read", "list"] +} +path "sys/mounts/*" { + capabilities = ["create", "update", "read"] +} +path "sys/policies/acl/*" { + capabilities = ["create", "update", "read"] +} +path "auth/kubernetes/config" { + capabilities = ["create", "update", "read"] +} +path "auth/kubernetes/role/*" { + capabilities = ["create", "update", "read"] +} +path "database/config/*" { + capabilities = ["create", "update", "read"] +} +path "database/roles/*" { + capabilities = ["create", "update", "read"] +} +path "rabbitmq/config/*" { + capabilities = ["create", "update", "read"] +} +path "rabbitmq/roles/*" { + capabilities = ["create", "update", "read"] +} +path "transit/keys/*" { + capabilities = ["create", "update", "read"] +} +path "kvv2/data/*" { + capabilities = ["create", "update", "read"] +} +path "kvv2/metadata/*" { + capabilities = ["create", "update", "read", "list", "delete"] +} +# Engine admin credentials the bootstrap reads to configure database/ and +# rabbitmq/ above. Read-only: the bootstrap never writes these. +path "secret/data/platform/postgres" { + capabilities = ["read"] +} +path "secret/data/platform/rabbitmq" { + capabilities = ["read"] +} +HCL +printf ' policy written\n' + +say "creating kubernetes auth role $ROLE" +# Bound to this one ServiceAccount in this one namespace. The TTL only has to +# outlast a single bootstrap run. +vault write "auth/kubernetes/role/${ROLE}" \ + bound_service_account_names="$SA" \ + bound_service_account_namespaces="$NS" \ + token_policies="$POLICY" \ + ttl=20m \ + max_ttl=1h >/dev/null +printf ' role bound to %s/%s, ttl=20m\n' "$NS" "$SA" + +say "verifying the ServiceAccount can log in" +# The only verification that counts: log in from inside the cluster as that SA, +# the same way the Job will. +k delete pod vault-bootstrap-login-probe -n "$NS" --ignore-not-found >/dev/null 2>&1 +cat </dev/null +apiVersion: v1 +kind: Pod +metadata: + name: vault-bootstrap-login-probe + namespace: ${NS} +spec: + serviceAccountName: ${SA} + restartPolicy: Never + nodeSelector: + platform.jorisjonkers.dev/site: frankfurt + containers: + - name: probe + image: hashicorp/vault:1.21.2 + env: + - name: VAULT_ADDR + value: http://vault.data-system.svc.cluster.local:8200 + command: ['/bin/sh','-ec'] + args: + - | + VAULT_TOKEN="\$(vault write -field=token auth/kubernetes/login \\ + role=${ROLE} \\ + jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token)" + export VAULT_TOKEN + echo "login: ok" + echo "policies: \$(vault token lookup -format=json | grep -o '"${POLICY}"' | head -1)" + vault policy read vso >/dev/null && echo "policy read: ok" + vault read -field=bound_service_account_names auth/kubernetes/role/vso >/dev/null && echo "role read: ok" +EOF +phase="" +for _ in $(seq 1 40); do + phase="$(k get pod vault-bootstrap-login-probe -n "$NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + case "$phase" in Succeeded|Failed) break;; esac + sleep 3 +done +printf ' probe phase: %s\n' "$phase" +k logs vault-bootstrap-login-probe -n "$NS" 2>&1 | sed 's/^/ /' +[ "$phase" = "Succeeded" ] || die "the ServiceAccount could not log in with role $ROLE" + +say "done" +printf 'Policy %s and kubernetes auth role %s exist, and %s/%s can log in.\n' \ + "$POLICY" "$ROLE" "$NS" "$SA" +printf '\nThe manifest change is already live (fleet-infra #128), so the Job only\n' +printf 'needs a re-run:\n' +printf ' kubectl --context %s delete job vault-bootstrap-auth -n %s\n' "$CTX" "$NS" +printf ' flux --context %s reconcile kustomization apps-data --with-source\n' "$CTX" +printf '\nThen confirm the roles and policies it was never able to write have landed:\n' +printf ' kubectl --context %s get job vault-bootstrap-auth -n %s\n' "$CTX" "$NS" +printf ' kubectl --context %s get vaultstaticsecret -n observability\n' "$CTX" +printf '\nThe vault-bootstrap-token Secret is then unused and can be deleted.\n' diff --git a/infra/scripts/flip-flux-source-to-monorepo.sh b/infra/scripts/flip-flux-source-to-monorepo.sh new file mode 100755 index 00000000..5063c408 --- /dev/null +++ b/infra/scripts/flip-flux-source-to-monorepo.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +############################################################################### +# Roll the live Flux source back to ExtraToast/personal-stack @ main. +# +# Three steps, not two. kubectl apply does NOT remove spec.provider, so +# applying the monorepo manifest while provider: github lingers leaves the +# source rejecting itself with the inverse error: +# "secretRef with github app data must be specified when provider is github" +# That is what happened on the first rollback, so the field is deleted +# explicitly here. +# +# Usage: bash infra/scripts/flip-flux-source-to-monorepo.sh +# MONOREPO=/path/to/personal-stack bash flip-flux-source-to-monorepo.sh +############################################################################### +set -uo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +# infra/scripts/ -> repo root is two levels up. Overridable for a clone +# checked out elsewhere. +MONOREPO="${MONOREPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +EXPECT_URL="https://github.com/ExtraToast/personal-stack" +SYNC="platform/cluster/flux/clusters/production/flux-system/gotk-sync.yaml" + +k() { kubectl --context "$CTX" "$@"; } +die() { echo "ABORT: $*" >&2; exit 1; } + +echo "== 1. verify the monorepo manifest ==" +[ -f "$MONOREPO/$SYNC" ] || die "$MONOREPO/$SYNC not found (set MONOREPO=...)" +grep -q "$EXPECT_URL" "$MONOREPO/$SYNC" || die "$SYNC does not point at $EXPECT_URL" +grep -q "provider:" "$MONOREPO/$SYNC" && die "$SYNC unexpectedly sets provider; review before rolling back" +echo " points at $EXPECT_URL, no provider field" + +echo "== 2. drop the GitHub App credentials ==" +# Ignore failures: the keys may already be absent, which is fine. +k patch secret flux-system -n flux-system --type=json -p '[ + {"op":"remove","path":"/data/githubAppID"}, + {"op":"remove","path":"/data/githubAppInstallationID"}, + {"op":"remove","path":"/data/githubAppPrivateKey"}]' 2>/dev/null \ + && echo " removed" || echo " already absent" + +echo "== 3. drop spec.provider from the live GitRepository ==" +k patch gitrepository flux-system -n flux-system --type=json \ + -p '[{"op":"remove","path":"/spec/provider"}]' 2>/dev/null \ + && echo " removed" || echo " already absent" + +echo "== 4. apply the monorepo source ==" +cd "$MONOREPO" || die "cannot cd $MONOREPO" +k apply -k platform/cluster/flux/clusters/production/flux-system || die "apply failed" + +echo "== 5. reconcile ==" +flux --context "$CTX" reconcile source git flux-system -n flux-system --timeout=180s || true +flux --context "$CTX" reconcile kustomization flux-system -n flux-system --timeout=300s || true + +echo "== 6. verify ==" +k get gitrepository -n flux-system flux-system \ + -o jsonpath=' url={.spec.url}{"\n"} branch={.spec.ref.branch}{"\n"} provider={.spec.provider}{"\n"} ready={.status.conditions[?(@.type=="Ready")].status}{"\n"} artifact={.status.artifact.revision}{"\n"}' +echo +echo " Reconciling the services most likely to have been mid-roll:" +for ks in apps-mail apps-utility-system apps-stateless; do + flux --context "$CTX" reconcile kustomization "$ks" --timeout=180s >/dev/null 2>&1 \ + && echo " $ks reconciled" || echo " $ks FAILED" +done +echo " pods not Running/Completed:" +k get pods -A --no-headers | awk '$4!="Running" && $4!="Completed" {print " "$1"/"$2" "$4}' | head -15 +echo +echo "KNOWN: agents-api cannot roll back. The database is at schema 23, created" +echo "by v0.19.1; the monorepo image knows 20 migrations and exits instead of" +echo "running against a newer schema. Its healthy v0.19.1 pod keeps serving, but" +echo "the Deployment's desired state is the broken image." diff --git a/infra/scripts/flip-flux-source-to-org.sh b/infra/scripts/flip-flux-source-to-org.sh new file mode 100755 index 00000000..99482609 --- /dev/null +++ b/infra/scripts/flip-flux-source-to-org.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +############################################################################### +# Flip the live Flux source to JorisJonkers-dev/fleet-infra @ deploy/production. +# +# The secret and the GitRepository must change together: source-controller +# rejects a secret carrying GitHub App keys unless provider is github, and +# rejects provider github unless the keys are present. Either half alone leaves +# the source InvalidProviderConfiguration, so this does both back to back. +# +# Usage: bash infra/scripts/flip-flux-source-to-org.sh +# Undo: bash flip-flux-source-to-monorepo.sh +############################################################################### +set -uo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +APP_ID=3942418 +INSTALL_ID=142960823 +EXPECT_URL="https://github.com/JorisJonkers-dev/fleet-infra" +EXPECT_BRANCH="deploy/production" +WORK="${WORK:-/tmp/flip-org-$$}" + +k() { kubectl --context "$CTX" "$@"; } +die() { echo "ABORT: $*" >&2; exit 1; } + +echo "== 1. checkout deploy/production ==" +rm -rf "$WORK" +git clone -q --depth 1 --branch "$EXPECT_BRANCH" "$EXPECT_URL" "$WORK" \ + || die "cannot clone $EXPECT_URL ($EXPECT_BRANCH)" +cd "$WORK" || die "cannot cd $WORK" +echo " $(git log --oneline -1)" + +echo "== 2. verify the manifest before applying it ==" +SYNC=cluster/flux/clusters/production/flux-system/gotk-sync.yaml +[ -f "$SYNC" ] || die "$SYNC missing" +grep -q "provider: github" "$SYNC" || die "gotk-sync has no 'provider: github'; the App keys would be rejected" +grep -q "$EXPECT_URL" "$SYNC" || die "gotk-sync does not point at $EXPECT_URL" +grep -q "branch: $EXPECT_BRANCH" "$SYNC" || die "gotk-sync does not target $EXPECT_BRANCH" +echo " provider: github, url and branch as expected" + +# apps-edge pruning must be off for the flip: 38 public IngressRoutes move to +# other Kustomizations, nothing orders apps-edge after the ones adopting them, +# and a prune-first reconcile drops public routing until they catch up. +PRUNE=$(python3 -c " +import yaml +for d in yaml.safe_load_all(open('cluster/flux/clusters/production/kustomizations.yaml')): + if d and d.get('kind')=='Kustomization' and d['metadata']['name']=='apps-edge': + print(d['spec'].get('prune')) +" 2>/dev/null) +if [ "$PRUNE" = "False" ]; then + echo " apps-edge prune: false (route relocation is safe)" +else + echo " WARNING: apps-edge prune=$PRUNE. Public routes may drop briefly during the flip." +fi + +echo "== 3. add the GitHub App credentials to the flux-system secret ==" +KEY_B64=$(k get secret -n agents-system github-app -o jsonpath='{.data.private-key}' 2>/dev/null) +[ -n "$KEY_B64" ] || die "cannot read agents-system/github-app private-key" +ID_B64=$(printf '%s' "$APP_ID" | base64) +INST_B64=$(printf '%s' "$INSTALL_ID" | base64) +k patch secret flux-system -n flux-system --type merge \ + -p "{\"data\":{\"githubAppPrivateKey\":\"$KEY_B64\",\"githubAppID\":\"$ID_B64\",\"githubAppInstallationID\":\"$INST_B64\"}}" \ + || die "secret patch failed" +echo " NOTE: the live source is invalid from here until step 4 applies provider: github." + +echo "== 4. apply the new source ==" +k apply -k cluster/flux/clusters/production/flux-system || die "apply failed -- run flip-flux-source-to-monorepo.sh" + +echo "== 5. reconcile ==" +flux --context "$CTX" reconcile source git flux-system -n flux-system --timeout=180s || true +flux --context "$CTX" reconcile kustomization flux-system -n flux-system --timeout=300s || true + +echo "== 6. verify ==" +k get gitrepository -n flux-system flux-system \ + -o jsonpath=' url={.spec.url}{"\n"} branch={.spec.ref.branch}{"\n"} provider={.spec.provider}{"\n"} ready={.status.conditions[?(@.type=="Ready")].status}{"\n"} artifact={.status.artifact.revision}{"\n"}' +echo +echo " Kustomizations not ready:" +k get kustomization -n flux-system -o json \ + | python3 -c 'import json,sys +for k in json.load(sys.stdin)["items"]: + c=[x for x in (k.get("status") or {}).get("conditions") or [] if x["type"]=="Ready"] + if c and c[0]["status"]!="True": print(" ",k["metadata"]["name"],"-",c[0].get("reason"))' +echo " pods not Running/Completed:" +k get pods -A --no-headers | awk '$4!="Running" && $4!="Completed" {print " "$1"/"$2" "$4}' | head -15 +echo " PVCs (expect 24):" "$(k get pvc -A --no-headers | wc -l | tr -d ' ')" +echo +echo "Done. If this went wrong: bash flip-flux-source-to-monorepo.sh" diff --git a/infra/scripts/post-cutover-fixups.sh b/infra/scripts/post-cutover-fixups.sh new file mode 100755 index 00000000..c6770288 --- /dev/null +++ b/infra/scripts/post-cutover-fixups.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +############################################################################### +# The three things left after the Flux source cutover that need credentials or +# a decision, in one pass: +# +# 1. Discord webhook URL -> secret/platform/observability discord.alert_webhook_url +# Without it Alertmanager has a receiver with no destination. +# 2. Vault metrics token -> secret/platform/observability vault.prometheus_token +# Expired. While it is dead Vault's /sys/metrics scrape 403s, which means +# VaultSealed cannot fire: its input series simply does not exist. +# 3. The four agents CronJobs failing on the revoked Claude OAuth token. +# Optional, opt-in, see --suspend-agent-cronjobs. +# +# Nothing is echoed: secrets are read with a silent prompt and passed to vault +# without appearing in output or shell history. +# +# Usage: +# bash post-cutover-fixups.sh +# bash post-cutover-fixups.sh --suspend-agent-cronjobs +# SKIP_WEBHOOK=1 bash post-cutover-fixups.sh # only the metrics token +# SKIP_TOKEN=1 bash post-cutover-fixups.sh # only the webhook +############################################################################### +set -uo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +VAULT_LOCAL_PORT="${VAULT_LOCAL_PORT:-8200}" +PROM_LOCAL_PORT="${PROM_LOCAL_PORT:-19490}" +SUSPEND_CRONJOBS=0 +[ "${1:-}" = "--suspend-agent-cronjobs" ] && SUSPEND_CRONJOBS=1 + +k() { kubectl --context "$CTX" "$@"; } +PF_PIDS="" +cleanup() { for p in $PF_PIDS; do kill "$p" 2>/dev/null; done; } +trap cleanup EXIT + +echo "==============================================================" +echo " 0. port-forward Vault (forward-auth blocks the CLI directly)" +echo "==============================================================" +k port-forward -n data-system svc/vault "${VAULT_LOCAL_PORT}:8200" >/dev/null 2>&1 & +PF_PIDS="$PF_PIDS $!" +export VAULT_ADDR="http://127.0.0.1:${VAULT_LOCAL_PORT}" +# --retry-connrefused waits for the tunnel without a fixed sleep. +if ! curl -fsS --retry 20 --retry-connrefused --retry-delay 1 \ + "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200" >/dev/null 2>&1; then + echo " ABORT: cannot reach Vault at $VAULT_ADDR"; exit 1 +fi +echo " reachable at $VAULT_ADDR" + +if [ "$(curl -fsS "${VAULT_ADDR}/v1/sys/seal-status" | sed -n 's/.*"sealed":\([a-z]*\).*/\1/p')" = "true" ]; then + echo " ABORT: Vault is SEALED. Unseal it first (see unseal-vault.sh), then re-run." + exit 1 +fi +echo " unsealed" + +echo +echo "==============================================================" +echo " 1. authenticate" +echo "==============================================================" +if [ -n "${VAULT_TOKEN:-}" ]; then + echo " using VAULT_TOKEN from the environment" +else + echo " Paste a token with rights to write secret/platform/observability and" + echo " create tokens. The initial root token is in" + echo " vault-keys-personal-stack.txt in this repo (untracked)." + printf ' token: ' + read -rs VAULT_TOKEN; echo + export VAULT_TOKEN +fi +if ! vault token lookup >/dev/null 2>&1; then + echo " ABORT: token rejected by Vault"; exit 1 +fi +echo " accepted" + +if [ "${SKIP_WEBHOOK:-0}" != "1" ]; then + echo + echo "==============================================================" + echo " 2. Discord webhook" + echo "==============================================================" + echo " ROTATE FIRST: the previous URL was pasted into a chat transcript, so" + echo " treat it as burned. Delete that webhook in Discord (Server Settings" + echo " -> Integrations -> Webhooks) and create a new one." + printf ' new webhook URL (blank to skip): ' + read -rs WEBHOOK; echo + if [ -z "$WEBHOOK" ]; then + echo " skipped" + else + case "$WEBHOOK" in + https://discord.com/api/webhooks/*|https://discordapp.com/api/webhooks/*) ;; + *) echo " ABORT: that does not look like a Discord webhook URL"; exit 1 ;; + esac + vault kv patch secret/platform/observability "discord.alert_webhook_url=$WEBHOOK" >/dev/null \ + && echo " written to secret/platform/observability" \ + || { echo " FAILED to write"; exit 1; } + unset WEBHOOK + fi +fi + +if [ "${SKIP_TOKEN:-0}" != "1" ]; then + echo + echo "==============================================================" + echo " 3. Vault metrics token" + echo "==============================================================" + if vault policy read prometheus-metrics >/dev/null 2>&1; then + echo " policy prometheus-metrics exists" + else + echo " policy prometheus-metrics is MISSING -- creating it read-only on" + echo " the metrics endpoint only." + vault policy write prometheus-metrics - >/dev/null <<'POLICY' +path "sys/metrics" { + capabilities = ["read", "list"] +} +POLICY + echo " created" + fi + # -period makes this a periodic token: it renews indefinitely only if + # something renews it, and nothing does, so it dies every 30 days. That is + # what expired last time. VaultMetricsUnreachable now alerts when it happens. + NEW_TOKEN=$(vault token create -policy=prometheus-metrics -period=720h -format=json \ + | sed -n 's/.*"client_token": *"\([^"]*\)".*/\1/p') + if [ -z "$NEW_TOKEN" ]; then echo " FAILED to create a token"; exit 1; fi + vault kv patch secret/platform/observability "vault.prometheus_token=$NEW_TOKEN" >/dev/null \ + && echo " new token created and written" \ + || { echo " FAILED to write the token"; exit 1; } + unset NEW_TOKEN + + echo " forcing VSO to re-sync (it refreshes hourly otherwise)" + k delete secret -n data-system vault-prometheus-token >/dev/null 2>&1 || true + for _ in $(seq 1 30); do + k get secret -n data-system vault-prometheus-token >/dev/null 2>&1 && break + sleep 2 + done + k get secret -n data-system vault-prometheus-token >/dev/null 2>&1 \ + && echo " VSO re-created the secret" \ + || echo " WARNING: VSO has not re-created it yet; check the VaultStaticSecret" +fi + +if [ "$SUSPEND_CRONJOBS" = "1" ]; then + echo + echo "==============================================================" + echo " 4. suspend the agents CronJobs" + echo "==============================================================" + echo " These fail because the agent runtime's Claude OAuth token is revoked." + echo " Suspending stops the 6-hourly failures, and stops KubeJobFailed" + echo " paging Discord forever once delivery works. It also means Claude-backed" + echo " agent sessions and KB curation stay broken -- this hides the signal," + echo " it does not fix the cause." + for cj in agents-refresh-ping agents-kb-curator-triage agents-kb-curator-weekly agents-kb-install; do + k patch cronjob -n agents-system "$cj" -p '{"spec":{"suspend":true}}' >/dev/null 2>&1 \ + && echo " suspended $cj" || echo " could not suspend $cj" + done + echo " Note: apps-agents is Flux-managed, so a reconcile will unsuspend these." + echo " To make it stick, the CronJobs need removing in fleet-infra." +fi + +echo +echo "==============================================================" +echo " 5. verify" +echo "==============================================================" +LEN=$(k get secret -n observability alertmanager-discord -o jsonpath='{.data.webhook-url}' 2>/dev/null | base64 -d 2>/dev/null | wc -c | tr -d ' ') +echo " projected webhook-url length: ${LEN:-0} (0 means Alertmanager still has no destination)" + +k port-forward -n observability svc/metrics-stack-prometheus "${PROM_LOCAL_PORT}:9090" >/dev/null 2>&1 & +PF_PIDS="$PF_PIDS $!" +if curl -fsS --retry 20 --retry-connrefused --retry-delay 1 "http://localhost:${PROM_LOCAL_PORT}/-/healthy" >/dev/null 2>&1; then + DOWN=$(curl -s --get "http://localhost:${PROM_LOCAL_PORT}/api/v1/query" \ + --data-urlencode 'query=up{job="vault"} == 0' \ + | grep -o '"metric"' | wc -l | tr -d ' ') + echo " vault scrape targets still down: ${DOWN:-?} (0 means the metrics token works" + echo " and VaultSealed can evaluate again; Prometheus may need a scrape interval)" +else + echo " could not reach Prometheus to verify the scrape" +fi + +echo +echo "Remaining, deliberately not automated:" +echo " - Claude OAuth re-auth via the agents-login portal (browser flow)." +echo " - Both tokens above are static. Kubernetes auth is the durable fix for" +echo " the metrics token and for vault-raft-snapshot, which rots the same way." diff --git a/infra/scripts/resume-mail-provisioner.sh b/infra/scripts/resume-mail-provisioner.sh new file mode 100755 index 00000000..1a29f5da --- /dev/null +++ b/infra/scripts/resume-mail-provisioner.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Resumes apps-mail so the stalwart provisioning sidecar lands, then verifies it. +# +# The rollout is `Recreate` on an RWO PVC: the serving pod is torn down before +# the replacement starts, so mail is down for the rollout window. That is the +# only reason this is a separate, explicit step rather than part of the merge. +# +# On any failure this re-suspends apps-mail and rolls the Deployment back, so a +# bad outcome costs the rollout window and not the service. +set -euo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +NS=mail-system +DEPLOY=stalwart +KS=apps-mail +PF_PID="" + +k() { kubectl --context "$CTX" "$@"; } +say() { printf '\n== %s\n' "$*"; } +die() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } + +cleanup() { [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null || true; } +trap cleanup EXIT + +rollback() { + printf '\n!! verification failed -- rolling back\n' >&2 + k patch kustomization "$KS" -n flux-system --type=merge -p '{"spec":{"suspend":true}}' >/dev/null 2>&1 || true + k rollout undo "deploy/$DEPLOY" -n "$NS" >/dev/null 2>&1 || true + k rollout status "deploy/$DEPLOY" -n "$NS" --timeout=180s || true + printf '\napps-mail is suspended again and the Deployment is rolled back.\n' >&2 + printf 'Sidecar logs from the failed attempt:\n' >&2 + k logs "deploy/$DEPLOY" -n "$NS" -c stalwart-apply --tail=60 2>/dev/null >&2 || true + exit 1 +} + +say "preflight: context and cluster" +k config current-context >/dev/null || die "context $CTX unreachable" +printf ' context: %s\n' "$(k config current-context)" + +say "preflight: apps-mail must be suspended" +susp="$(k get kustomization "$KS" -n flux-system -o jsonpath='{.spec.suspend}')" +[ "$susp" = "true" ] || die "apps-mail is not suspended (suspend=$susp); refusing to guess at its state" +printf ' suspended: yes\n' + +say "preflight: the passwords the manifest references must resolve" +fail=0 +check_key() { # secret key + local len + len="$(k get secret "$1" -n "$NS" -o jsonpath="{.data.$2}" 2>/dev/null | base64 -d 2>/dev/null | wc -c | tr -d ' ')" + printf ' %-24s %-22s len=%s\n' "$1" "$2" "${len:-0}" + [ "${len:-0}" -gt 0 ] || fail=1 +} +check_key stalwart-auth-mail AUTH_MAIL_PASSWORD +check_key stalwart-mail ACCOUNT_MAIL_PASSWORD +[ "$fail" -eq 0 ] || die "a referenced password is empty; the manifest would be refused at validation" + +say "preflight: git revision must carry the sidecar" +rev="$(k get gitrepository flux-system -n flux-system -o jsonpath='{.status.artifact.revision}')" +printf ' source revision: %s\n' "$rev" + +say "capturing rollback point" +before_rev="$(k get deploy "$DEPLOY" -n "$NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')" +printf ' deployment revision before: %s\n' "$before_rev" + +say "resuming $KS" +k patch kustomization "$KS" -n flux-system --type=merge -p '{"spec":{"suspend":false}}' >/dev/null +flux --context "$CTX" reconcile kustomization "$KS" --with-source --timeout=180s || true + +say "waiting for the rollout" +if ! k rollout status "deploy/$DEPLOY" -n "$NS" --timeout=300s; then + rollback +fi + +say "verify: both containers ready" +for _ in $(seq 1 30); do + ready="$(k get pods -n "$NS" -l app.kubernetes.io/name=stalwart \ + -o jsonpath='{.items[0].status.containerStatuses[*].ready}' 2>/dev/null)" + case "$ready" in *false*|"") sleep 5;; *) break;; esac +done +printf ' container ready flags: %s\n' "$ready" +case "$ready" in *false*|"") rollback;; esac + +say "verify: the sidecar reconciled" +logs="$(k logs "deploy/$DEPLOY" -n "$NS" -c stalwart-apply --tail=200 2>/dev/null || true)" +printf '%s\n' "$logs" | grep -E 'catch-all|credentials|reconcile complete|FAIL' | sed 's/^/ /' || true +printf '%s' "$logs" | grep -q 'reconcile complete' || { printf ' no "reconcile complete" in the sidecar log\n' >&2; rollback; } +if printf '%s' "$logs" | grep -q '^FAIL'; then + printf ' sidecar reported FAIL\n' >&2 + rollback +fi + +say "verify: stalwart is serving" +k port-forward -n "$NS" "deploy/$DEPLOY" 18080:8080 >/dev/null 2>&1 & +PF_PID=$! +for _ in $(seq 1 20); do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 http://127.0.0.1:18080/admin/ 2>/dev/null || true)" + case "$code" in 200|301|302|401) break;; *) sleep 2;; esac +done +printf ' GET /admin/ -> %s\n' "${code:-no response}" +case "${code:-}" in 200|301|302|401) ;; *) rollback;; esac + +k port-forward -n "$NS" "deploy/$DEPLOY" 10143:143 >/dev/null 2>&1 & +imap_pid=$! +sleep 3 +if command -v nc >/dev/null 2>&1 && nc -z 127.0.0.1 10143 2>/dev/null; then + printf ' IMAP 143 accepts connections\n' +else + printf ' IMAP 143 check inconclusive (nc unavailable or refused)\n' +fi +kill "$imap_pid" 2>/dev/null || true + +say "done" +printf 'apps-mail is resumed, the sidecar reconciled, and stalwart is serving.\n' +printf 'joris.jonkers and n8n remain unmanaged by design -- Vault holds no password\n' +printf 'for either. Populate secret/platform/mail joris.password and n8n.password,\n' +printf 'then move them into managedAccounts, to bring them under reconciliation.\n'