diff --git a/infra/scripts/fix-vault-bootstrap-auth.sh b/infra/scripts/fix-vault-bootstrap-auth.sh new file mode 100755 index 00000000..b225b63f --- /dev/null +++ b/infra/scripts/fix-vault-bootstrap-auth.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# One-time: give vault-bootstrap-auth a Kubernetes auth role so it stops +# depending on a static token that expires unnoticed. +# +# The Job authenticates with a token from the vault-bootstrap-token Secret. That +# token is invalid (`vault token lookup` -> 403), so the Job has been failing, +# its pods are reaped, and every auth role and policy added to bootstrap-auth.sh +# since it expired was applied by Flux and never reached the server. Three +# backups are blocked on exactly that. +# +# This needs a privileged Vault token ONCE. Afterwards the Job logs in with its +# ServiceAccount and there is nothing left to expire. +# +# The companion manifest change is already merged (fleet-infra #128), so the Job +# currently fails with `invalid role name "vault-bootstrap"` -- this script is +# what creates that role. It only adds a policy and a role; it changes nothing +# that is currently working. +set -euo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +NS=data-system +SA=vault-bootstrap-auth +ROLE=vault-bootstrap +POLICY=vault-bootstrap +PORT=18200 +PF_PID="" + +k() { kubectl --context "$CTX" "$@"; } +say() { printf '\n== %s\n' "$*"; } +die() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } +cleanup() { + [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null || true + k delete pod vault-bootstrap-login-probe -n "$NS" --ignore-not-found >/dev/null 2>&1 || true +} +trap cleanup EXIT + +say "preflight" +k config current-context >/dev/null || die "context $CTX unreachable" +printf ' context: %s\n' "$(k config current-context)" +k get sa "$SA" -n "$NS" >/dev/null 2>&1 || die "ServiceAccount $NS/$SA not found" +printf ' serviceaccount: %s/%s\n' "$NS" "$SA" + +# Vault's TokenReview call needs the SA bound to system:auth-delegator. The Job +# manifest already installs that binding; check rather than assume. +if k get clusterrolebinding vault-bootstrap-auth-token-reviewer >/dev/null 2>&1; then + printf ' token-reviewer binding: present\n' +else + printf ' token-reviewer binding: MISSING -- kubernetes auth login will fail\n' >&2 + die "apply cluster/flux/apps/data/vault/bootstrap-auth-job.yaml first" +fi + +say "opening a port-forward to vault" +k port-forward -n "$NS" svc/vault "$PORT":8200 >/dev/null 2>&1 & +PF_PID=$! +export VAULT_ADDR="http://127.0.0.1:${PORT}" +ready=no +for _ in $(seq 1 20); do + if curl -fsS --max-time 2 "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200" >/dev/null 2>&1; then + ready=yes; break + fi + sleep 2 +done +[ "$ready" = yes ] || die "vault is not reachable on $VAULT_ADDR" + +sealed="$(curl -fsS "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200" | sed -n 's/.*"sealed":\([a-z]*\).*/\1/p')" +printf ' sealed: %s\n' "$sealed" +[ "$sealed" = "false" ] || die "vault is sealed -- unseal before bootstrapping auth" + +say "privileged token" +printf 'Paste a Vault token allowed to write policies and auth roles.\n' +printf 'The initial root token works. Input is hidden.\n' +printf 'token: ' +read -rs VAULT_TOKEN +printf '\n' +export VAULT_TOKEN +[ -n "$VAULT_TOKEN" ] || die "no token provided" + +command -v vault >/dev/null 2>&1 \ + || die "the vault CLI is not installed locally; install it and re-run" +vault token lookup >/dev/null 2>&1 || die "that token is not valid against $VAULT_ADDR" +printf ' token accepted\n' + +say "writing policy $POLICY" +# Scoped to what bootstrap-auth.sh actually does: enable the kubernetes auth +# method and the kvv2/database/rabbitmq engines, write the policies and roles it +# defines, manage the auth-api transit signing key, and read the engine admin +# credentials it configures those engines with. +vault policy write "$POLICY" - <<'HCL' +path "sys/auth" { + capabilities = ["read", "list"] +} +path "sys/auth/*" { + capabilities = ["create", "update", "read", "sudo"] +} +path "sys/mounts" { + capabilities = ["read", "list"] +} +path "sys/mounts/*" { + capabilities = ["create", "update", "read"] +} +path "sys/policies/acl/*" { + capabilities = ["create", "update", "read"] +} +path "auth/kubernetes/config" { + capabilities = ["create", "update", "read"] +} +path "auth/kubernetes/role/*" { + capabilities = ["create", "update", "read"] +} +path "database/config/*" { + capabilities = ["create", "update", "read"] +} +path "database/roles/*" { + capabilities = ["create", "update", "read"] +} +path "rabbitmq/config/*" { + capabilities = ["create", "update", "read"] +} +path "rabbitmq/roles/*" { + capabilities = ["create", "update", "read"] +} +path "transit/keys/*" { + capabilities = ["create", "update", "read"] +} +path "kvv2/data/*" { + capabilities = ["create", "update", "read"] +} +path "kvv2/metadata/*" { + capabilities = ["create", "update", "read", "list", "delete"] +} +# Engine admin credentials the bootstrap reads to configure database/ and +# rabbitmq/ above. Read-only: the bootstrap never writes these. +path "secret/data/platform/postgres" { + capabilities = ["read"] +} +path "secret/data/platform/rabbitmq" { + capabilities = ["read"] +} +HCL +printf ' policy written\n' + +say "creating kubernetes auth role $ROLE" +# Bound to this one ServiceAccount in this one namespace. The TTL only has to +# outlast a single bootstrap run. +vault write "auth/kubernetes/role/${ROLE}" \ + bound_service_account_names="$SA" \ + bound_service_account_namespaces="$NS" \ + token_policies="$POLICY" \ + ttl=20m \ + max_ttl=1h >/dev/null +printf ' role bound to %s/%s, ttl=20m\n' "$NS" "$SA" + +say "verifying the ServiceAccount can log in" +# The only verification that counts: log in from inside the cluster as that SA, +# the same way the Job will. +k delete pod vault-bootstrap-login-probe -n "$NS" --ignore-not-found >/dev/null 2>&1 +cat </dev/null +apiVersion: v1 +kind: Pod +metadata: + name: vault-bootstrap-login-probe + namespace: ${NS} +spec: + serviceAccountName: ${SA} + restartPolicy: Never + nodeSelector: + platform.jorisjonkers.dev/site: frankfurt + containers: + - name: probe + image: hashicorp/vault:1.21.2 + env: + - name: VAULT_ADDR + value: http://vault.data-system.svc.cluster.local:8200 + command: ['/bin/sh','-ec'] + args: + - | + VAULT_TOKEN="\$(vault write -field=token auth/kubernetes/login \\ + role=${ROLE} \\ + jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token)" + export VAULT_TOKEN + echo "login: ok" + echo "policies: \$(vault token lookup -format=json | grep -o '"${POLICY}"' | head -1)" + vault policy read vso >/dev/null && echo "policy read: ok" + vault read -field=bound_service_account_names auth/kubernetes/role/vso >/dev/null && echo "role read: ok" +EOF +phase="" +for _ in $(seq 1 40); do + phase="$(k get pod vault-bootstrap-login-probe -n "$NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + case "$phase" in Succeeded|Failed) break;; esac + sleep 3 +done +printf ' probe phase: %s\n' "$phase" +k logs vault-bootstrap-login-probe -n "$NS" 2>&1 | sed 's/^/ /' +[ "$phase" = "Succeeded" ] || die "the ServiceAccount could not log in with role $ROLE" + +say "done" +printf 'Policy %s and kubernetes auth role %s exist, and %s/%s can log in.\n' \ + "$POLICY" "$ROLE" "$NS" "$SA" +printf '\nThe manifest change is already live (fleet-infra #128), so the Job only\n' +printf 'needs a re-run:\n' +printf ' kubectl --context %s delete job vault-bootstrap-auth -n %s\n' "$CTX" "$NS" +printf ' flux --context %s reconcile kustomization apps-data --with-source\n' "$CTX" +printf '\nThen confirm the roles and policies it was never able to write have landed:\n' +printf ' kubectl --context %s get job vault-bootstrap-auth -n %s\n' "$CTX" "$NS" +printf ' kubectl --context %s get vaultstaticsecret -n observability\n' "$CTX" +printf '\nThe vault-bootstrap-token Secret is then unused and can be deleted.\n' diff --git a/infra/scripts/flip-flux-source-to-monorepo.sh b/infra/scripts/flip-flux-source-to-monorepo.sh new file mode 100755 index 00000000..5063c408 --- /dev/null +++ b/infra/scripts/flip-flux-source-to-monorepo.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +############################################################################### +# Roll the live Flux source back to ExtraToast/personal-stack @ main. +# +# Three steps, not two. kubectl apply does NOT remove spec.provider, so +# applying the monorepo manifest while provider: github lingers leaves the +# source rejecting itself with the inverse error: +# "secretRef with github app data must be specified when provider is github" +# That is what happened on the first rollback, so the field is deleted +# explicitly here. +# +# Usage: bash infra/scripts/flip-flux-source-to-monorepo.sh +# MONOREPO=/path/to/personal-stack bash flip-flux-source-to-monorepo.sh +############################################################################### +set -uo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +# infra/scripts/ -> repo root is two levels up. Overridable for a clone +# checked out elsewhere. +MONOREPO="${MONOREPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +EXPECT_URL="https://github.com/ExtraToast/personal-stack" +SYNC="platform/cluster/flux/clusters/production/flux-system/gotk-sync.yaml" + +k() { kubectl --context "$CTX" "$@"; } +die() { echo "ABORT: $*" >&2; exit 1; } + +echo "== 1. verify the monorepo manifest ==" +[ -f "$MONOREPO/$SYNC" ] || die "$MONOREPO/$SYNC not found (set MONOREPO=...)" +grep -q "$EXPECT_URL" "$MONOREPO/$SYNC" || die "$SYNC does not point at $EXPECT_URL" +grep -q "provider:" "$MONOREPO/$SYNC" && die "$SYNC unexpectedly sets provider; review before rolling back" +echo " points at $EXPECT_URL, no provider field" + +echo "== 2. drop the GitHub App credentials ==" +# Ignore failures: the keys may already be absent, which is fine. +k patch secret flux-system -n flux-system --type=json -p '[ + {"op":"remove","path":"/data/githubAppID"}, + {"op":"remove","path":"/data/githubAppInstallationID"}, + {"op":"remove","path":"/data/githubAppPrivateKey"}]' 2>/dev/null \ + && echo " removed" || echo " already absent" + +echo "== 3. drop spec.provider from the live GitRepository ==" +k patch gitrepository flux-system -n flux-system --type=json \ + -p '[{"op":"remove","path":"/spec/provider"}]' 2>/dev/null \ + && echo " removed" || echo " already absent" + +echo "== 4. apply the monorepo source ==" +cd "$MONOREPO" || die "cannot cd $MONOREPO" +k apply -k platform/cluster/flux/clusters/production/flux-system || die "apply failed" + +echo "== 5. reconcile ==" +flux --context "$CTX" reconcile source git flux-system -n flux-system --timeout=180s || true +flux --context "$CTX" reconcile kustomization flux-system -n flux-system --timeout=300s || true + +echo "== 6. verify ==" +k get gitrepository -n flux-system flux-system \ + -o jsonpath=' url={.spec.url}{"\n"} branch={.spec.ref.branch}{"\n"} provider={.spec.provider}{"\n"} ready={.status.conditions[?(@.type=="Ready")].status}{"\n"} artifact={.status.artifact.revision}{"\n"}' +echo +echo " Reconciling the services most likely to have been mid-roll:" +for ks in apps-mail apps-utility-system apps-stateless; do + flux --context "$CTX" reconcile kustomization "$ks" --timeout=180s >/dev/null 2>&1 \ + && echo " $ks reconciled" || echo " $ks FAILED" +done +echo " pods not Running/Completed:" +k get pods -A --no-headers | awk '$4!="Running" && $4!="Completed" {print " "$1"/"$2" "$4}' | head -15 +echo +echo "KNOWN: agents-api cannot roll back. The database is at schema 23, created" +echo "by v0.19.1; the monorepo image knows 20 migrations and exits instead of" +echo "running against a newer schema. Its healthy v0.19.1 pod keeps serving, but" +echo "the Deployment's desired state is the broken image." diff --git a/infra/scripts/flip-flux-source-to-org.sh b/infra/scripts/flip-flux-source-to-org.sh new file mode 100755 index 00000000..99482609 --- /dev/null +++ b/infra/scripts/flip-flux-source-to-org.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +############################################################################### +# Flip the live Flux source to JorisJonkers-dev/fleet-infra @ deploy/production. +# +# The secret and the GitRepository must change together: source-controller +# rejects a secret carrying GitHub App keys unless provider is github, and +# rejects provider github unless the keys are present. Either half alone leaves +# the source InvalidProviderConfiguration, so this does both back to back. +# +# Usage: bash infra/scripts/flip-flux-source-to-org.sh +# Undo: bash flip-flux-source-to-monorepo.sh +############################################################################### +set -uo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +APP_ID=3942418 +INSTALL_ID=142960823 +EXPECT_URL="https://github.com/JorisJonkers-dev/fleet-infra" +EXPECT_BRANCH="deploy/production" +WORK="${WORK:-/tmp/flip-org-$$}" + +k() { kubectl --context "$CTX" "$@"; } +die() { echo "ABORT: $*" >&2; exit 1; } + +echo "== 1. checkout deploy/production ==" +rm -rf "$WORK" +git clone -q --depth 1 --branch "$EXPECT_BRANCH" "$EXPECT_URL" "$WORK" \ + || die "cannot clone $EXPECT_URL ($EXPECT_BRANCH)" +cd "$WORK" || die "cannot cd $WORK" +echo " $(git log --oneline -1)" + +echo "== 2. verify the manifest before applying it ==" +SYNC=cluster/flux/clusters/production/flux-system/gotk-sync.yaml +[ -f "$SYNC" ] || die "$SYNC missing" +grep -q "provider: github" "$SYNC" || die "gotk-sync has no 'provider: github'; the App keys would be rejected" +grep -q "$EXPECT_URL" "$SYNC" || die "gotk-sync does not point at $EXPECT_URL" +grep -q "branch: $EXPECT_BRANCH" "$SYNC" || die "gotk-sync does not target $EXPECT_BRANCH" +echo " provider: github, url and branch as expected" + +# apps-edge pruning must be off for the flip: 38 public IngressRoutes move to +# other Kustomizations, nothing orders apps-edge after the ones adopting them, +# and a prune-first reconcile drops public routing until they catch up. +PRUNE=$(python3 -c " +import yaml +for d in yaml.safe_load_all(open('cluster/flux/clusters/production/kustomizations.yaml')): + if d and d.get('kind')=='Kustomization' and d['metadata']['name']=='apps-edge': + print(d['spec'].get('prune')) +" 2>/dev/null) +if [ "$PRUNE" = "False" ]; then + echo " apps-edge prune: false (route relocation is safe)" +else + echo " WARNING: apps-edge prune=$PRUNE. Public routes may drop briefly during the flip." +fi + +echo "== 3. add the GitHub App credentials to the flux-system secret ==" +KEY_B64=$(k get secret -n agents-system github-app -o jsonpath='{.data.private-key}' 2>/dev/null) +[ -n "$KEY_B64" ] || die "cannot read agents-system/github-app private-key" +ID_B64=$(printf '%s' "$APP_ID" | base64) +INST_B64=$(printf '%s' "$INSTALL_ID" | base64) +k patch secret flux-system -n flux-system --type merge \ + -p "{\"data\":{\"githubAppPrivateKey\":\"$KEY_B64\",\"githubAppID\":\"$ID_B64\",\"githubAppInstallationID\":\"$INST_B64\"}}" \ + || die "secret patch failed" +echo " NOTE: the live source is invalid from here until step 4 applies provider: github." + +echo "== 4. apply the new source ==" +k apply -k cluster/flux/clusters/production/flux-system || die "apply failed -- run flip-flux-source-to-monorepo.sh" + +echo "== 5. reconcile ==" +flux --context "$CTX" reconcile source git flux-system -n flux-system --timeout=180s || true +flux --context "$CTX" reconcile kustomization flux-system -n flux-system --timeout=300s || true + +echo "== 6. verify ==" +k get gitrepository -n flux-system flux-system \ + -o jsonpath=' url={.spec.url}{"\n"} branch={.spec.ref.branch}{"\n"} provider={.spec.provider}{"\n"} ready={.status.conditions[?(@.type=="Ready")].status}{"\n"} artifact={.status.artifact.revision}{"\n"}' +echo +echo " Kustomizations not ready:" +k get kustomization -n flux-system -o json \ + | python3 -c 'import json,sys +for k in json.load(sys.stdin)["items"]: + c=[x for x in (k.get("status") or {}).get("conditions") or [] if x["type"]=="Ready"] + if c and c[0]["status"]!="True": print(" ",k["metadata"]["name"],"-",c[0].get("reason"))' +echo " pods not Running/Completed:" +k get pods -A --no-headers | awk '$4!="Running" && $4!="Completed" {print " "$1"/"$2" "$4}' | head -15 +echo " PVCs (expect 24):" "$(k get pvc -A --no-headers | wc -l | tr -d ' ')" +echo +echo "Done. If this went wrong: bash flip-flux-source-to-monorepo.sh" diff --git a/infra/scripts/post-cutover-fixups.sh b/infra/scripts/post-cutover-fixups.sh new file mode 100755 index 00000000..c6770288 --- /dev/null +++ b/infra/scripts/post-cutover-fixups.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +############################################################################### +# The three things left after the Flux source cutover that need credentials or +# a decision, in one pass: +# +# 1. Discord webhook URL -> secret/platform/observability discord.alert_webhook_url +# Without it Alertmanager has a receiver with no destination. +# 2. Vault metrics token -> secret/platform/observability vault.prometheus_token +# Expired. While it is dead Vault's /sys/metrics scrape 403s, which means +# VaultSealed cannot fire: its input series simply does not exist. +# 3. The four agents CronJobs failing on the revoked Claude OAuth token. +# Optional, opt-in, see --suspend-agent-cronjobs. +# +# Nothing is echoed: secrets are read with a silent prompt and passed to vault +# without appearing in output or shell history. +# +# Usage: +# bash post-cutover-fixups.sh +# bash post-cutover-fixups.sh --suspend-agent-cronjobs +# SKIP_WEBHOOK=1 bash post-cutover-fixups.sh # only the metrics token +# SKIP_TOKEN=1 bash post-cutover-fixups.sh # only the webhook +############################################################################### +set -uo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +VAULT_LOCAL_PORT="${VAULT_LOCAL_PORT:-8200}" +PROM_LOCAL_PORT="${PROM_LOCAL_PORT:-19490}" +SUSPEND_CRONJOBS=0 +[ "${1:-}" = "--suspend-agent-cronjobs" ] && SUSPEND_CRONJOBS=1 + +k() { kubectl --context "$CTX" "$@"; } +PF_PIDS="" +cleanup() { for p in $PF_PIDS; do kill "$p" 2>/dev/null; done; } +trap cleanup EXIT + +echo "==============================================================" +echo " 0. port-forward Vault (forward-auth blocks the CLI directly)" +echo "==============================================================" +k port-forward -n data-system svc/vault "${VAULT_LOCAL_PORT}:8200" >/dev/null 2>&1 & +PF_PIDS="$PF_PIDS $!" +export VAULT_ADDR="http://127.0.0.1:${VAULT_LOCAL_PORT}" +# --retry-connrefused waits for the tunnel without a fixed sleep. +if ! curl -fsS --retry 20 --retry-connrefused --retry-delay 1 \ + "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200" >/dev/null 2>&1; then + echo " ABORT: cannot reach Vault at $VAULT_ADDR"; exit 1 +fi +echo " reachable at $VAULT_ADDR" + +if [ "$(curl -fsS "${VAULT_ADDR}/v1/sys/seal-status" | sed -n 's/.*"sealed":\([a-z]*\).*/\1/p')" = "true" ]; then + echo " ABORT: Vault is SEALED. Unseal it first (see unseal-vault.sh), then re-run." + exit 1 +fi +echo " unsealed" + +echo +echo "==============================================================" +echo " 1. authenticate" +echo "==============================================================" +if [ -n "${VAULT_TOKEN:-}" ]; then + echo " using VAULT_TOKEN from the environment" +else + echo " Paste a token with rights to write secret/platform/observability and" + echo " create tokens. The initial root token is in" + echo " vault-keys-personal-stack.txt in this repo (untracked)." + printf ' token: ' + read -rs VAULT_TOKEN; echo + export VAULT_TOKEN +fi +if ! vault token lookup >/dev/null 2>&1; then + echo " ABORT: token rejected by Vault"; exit 1 +fi +echo " accepted" + +if [ "${SKIP_WEBHOOK:-0}" != "1" ]; then + echo + echo "==============================================================" + echo " 2. Discord webhook" + echo "==============================================================" + echo " ROTATE FIRST: the previous URL was pasted into a chat transcript, so" + echo " treat it as burned. Delete that webhook in Discord (Server Settings" + echo " -> Integrations -> Webhooks) and create a new one." + printf ' new webhook URL (blank to skip): ' + read -rs WEBHOOK; echo + if [ -z "$WEBHOOK" ]; then + echo " skipped" + else + case "$WEBHOOK" in + https://discord.com/api/webhooks/*|https://discordapp.com/api/webhooks/*) ;; + *) echo " ABORT: that does not look like a Discord webhook URL"; exit 1 ;; + esac + vault kv patch secret/platform/observability "discord.alert_webhook_url=$WEBHOOK" >/dev/null \ + && echo " written to secret/platform/observability" \ + || { echo " FAILED to write"; exit 1; } + unset WEBHOOK + fi +fi + +if [ "${SKIP_TOKEN:-0}" != "1" ]; then + echo + echo "==============================================================" + echo " 3. Vault metrics token" + echo "==============================================================" + if vault policy read prometheus-metrics >/dev/null 2>&1; then + echo " policy prometheus-metrics exists" + else + echo " policy prometheus-metrics is MISSING -- creating it read-only on" + echo " the metrics endpoint only." + vault policy write prometheus-metrics - >/dev/null <<'POLICY' +path "sys/metrics" { + capabilities = ["read", "list"] +} +POLICY + echo " created" + fi + # -period makes this a periodic token: it renews indefinitely only if + # something renews it, and nothing does, so it dies every 30 days. That is + # what expired last time. VaultMetricsUnreachable now alerts when it happens. + NEW_TOKEN=$(vault token create -policy=prometheus-metrics -period=720h -format=json \ + | sed -n 's/.*"client_token": *"\([^"]*\)".*/\1/p') + if [ -z "$NEW_TOKEN" ]; then echo " FAILED to create a token"; exit 1; fi + vault kv patch secret/platform/observability "vault.prometheus_token=$NEW_TOKEN" >/dev/null \ + && echo " new token created and written" \ + || { echo " FAILED to write the token"; exit 1; } + unset NEW_TOKEN + + echo " forcing VSO to re-sync (it refreshes hourly otherwise)" + k delete secret -n data-system vault-prometheus-token >/dev/null 2>&1 || true + for _ in $(seq 1 30); do + k get secret -n data-system vault-prometheus-token >/dev/null 2>&1 && break + sleep 2 + done + k get secret -n data-system vault-prometheus-token >/dev/null 2>&1 \ + && echo " VSO re-created the secret" \ + || echo " WARNING: VSO has not re-created it yet; check the VaultStaticSecret" +fi + +if [ "$SUSPEND_CRONJOBS" = "1" ]; then + echo + echo "==============================================================" + echo " 4. suspend the agents CronJobs" + echo "==============================================================" + echo " These fail because the agent runtime's Claude OAuth token is revoked." + echo " Suspending stops the 6-hourly failures, and stops KubeJobFailed" + echo " paging Discord forever once delivery works. It also means Claude-backed" + echo " agent sessions and KB curation stay broken -- this hides the signal," + echo " it does not fix the cause." + for cj in agents-refresh-ping agents-kb-curator-triage agents-kb-curator-weekly agents-kb-install; do + k patch cronjob -n agents-system "$cj" -p '{"spec":{"suspend":true}}' >/dev/null 2>&1 \ + && echo " suspended $cj" || echo " could not suspend $cj" + done + echo " Note: apps-agents is Flux-managed, so a reconcile will unsuspend these." + echo " To make it stick, the CronJobs need removing in fleet-infra." +fi + +echo +echo "==============================================================" +echo " 5. verify" +echo "==============================================================" +LEN=$(k get secret -n observability alertmanager-discord -o jsonpath='{.data.webhook-url}' 2>/dev/null | base64 -d 2>/dev/null | wc -c | tr -d ' ') +echo " projected webhook-url length: ${LEN:-0} (0 means Alertmanager still has no destination)" + +k port-forward -n observability svc/metrics-stack-prometheus "${PROM_LOCAL_PORT}:9090" >/dev/null 2>&1 & +PF_PIDS="$PF_PIDS $!" +if curl -fsS --retry 20 --retry-connrefused --retry-delay 1 "http://localhost:${PROM_LOCAL_PORT}/-/healthy" >/dev/null 2>&1; then + DOWN=$(curl -s --get "http://localhost:${PROM_LOCAL_PORT}/api/v1/query" \ + --data-urlencode 'query=up{job="vault"} == 0' \ + | grep -o '"metric"' | wc -l | tr -d ' ') + echo " vault scrape targets still down: ${DOWN:-?} (0 means the metrics token works" + echo " and VaultSealed can evaluate again; Prometheus may need a scrape interval)" +else + echo " could not reach Prometheus to verify the scrape" +fi + +echo +echo "Remaining, deliberately not automated:" +echo " - Claude OAuth re-auth via the agents-login portal (browser flow)." +echo " - Both tokens above are static. Kubernetes auth is the durable fix for" +echo " the metrics token and for vault-raft-snapshot, which rots the same way." diff --git a/infra/scripts/resume-mail-provisioner.sh b/infra/scripts/resume-mail-provisioner.sh new file mode 100755 index 00000000..1a29f5da --- /dev/null +++ b/infra/scripts/resume-mail-provisioner.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Resumes apps-mail so the stalwart provisioning sidecar lands, then verifies it. +# +# The rollout is `Recreate` on an RWO PVC: the serving pod is torn down before +# the replacement starts, so mail is down for the rollout window. That is the +# only reason this is a separate, explicit step rather than part of the merge. +# +# On any failure this re-suspends apps-mail and rolls the Deployment back, so a +# bad outcome costs the rollout window and not the service. +set -euo pipefail + +CTX="${KUBE_CONTEXT:-personal}" +NS=mail-system +DEPLOY=stalwart +KS=apps-mail +PF_PID="" + +k() { kubectl --context "$CTX" "$@"; } +say() { printf '\n== %s\n' "$*"; } +die() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } + +cleanup() { [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null || true; } +trap cleanup EXIT + +rollback() { + printf '\n!! verification failed -- rolling back\n' >&2 + k patch kustomization "$KS" -n flux-system --type=merge -p '{"spec":{"suspend":true}}' >/dev/null 2>&1 || true + k rollout undo "deploy/$DEPLOY" -n "$NS" >/dev/null 2>&1 || true + k rollout status "deploy/$DEPLOY" -n "$NS" --timeout=180s || true + printf '\napps-mail is suspended again and the Deployment is rolled back.\n' >&2 + printf 'Sidecar logs from the failed attempt:\n' >&2 + k logs "deploy/$DEPLOY" -n "$NS" -c stalwart-apply --tail=60 2>/dev/null >&2 || true + exit 1 +} + +say "preflight: context and cluster" +k config current-context >/dev/null || die "context $CTX unreachable" +printf ' context: %s\n' "$(k config current-context)" + +say "preflight: apps-mail must be suspended" +susp="$(k get kustomization "$KS" -n flux-system -o jsonpath='{.spec.suspend}')" +[ "$susp" = "true" ] || die "apps-mail is not suspended (suspend=$susp); refusing to guess at its state" +printf ' suspended: yes\n' + +say "preflight: the passwords the manifest references must resolve" +fail=0 +check_key() { # secret key + local len + len="$(k get secret "$1" -n "$NS" -o jsonpath="{.data.$2}" 2>/dev/null | base64 -d 2>/dev/null | wc -c | tr -d ' ')" + printf ' %-24s %-22s len=%s\n' "$1" "$2" "${len:-0}" + [ "${len:-0}" -gt 0 ] || fail=1 +} +check_key stalwart-auth-mail AUTH_MAIL_PASSWORD +check_key stalwart-mail ACCOUNT_MAIL_PASSWORD +[ "$fail" -eq 0 ] || die "a referenced password is empty; the manifest would be refused at validation" + +say "preflight: git revision must carry the sidecar" +rev="$(k get gitrepository flux-system -n flux-system -o jsonpath='{.status.artifact.revision}')" +printf ' source revision: %s\n' "$rev" + +say "capturing rollback point" +before_rev="$(k get deploy "$DEPLOY" -n "$NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')" +printf ' deployment revision before: %s\n' "$before_rev" + +say "resuming $KS" +k patch kustomization "$KS" -n flux-system --type=merge -p '{"spec":{"suspend":false}}' >/dev/null +flux --context "$CTX" reconcile kustomization "$KS" --with-source --timeout=180s || true + +say "waiting for the rollout" +if ! k rollout status "deploy/$DEPLOY" -n "$NS" --timeout=300s; then + rollback +fi + +say "verify: both containers ready" +for _ in $(seq 1 30); do + ready="$(k get pods -n "$NS" -l app.kubernetes.io/name=stalwart \ + -o jsonpath='{.items[0].status.containerStatuses[*].ready}' 2>/dev/null)" + case "$ready" in *false*|"") sleep 5;; *) break;; esac +done +printf ' container ready flags: %s\n' "$ready" +case "$ready" in *false*|"") rollback;; esac + +say "verify: the sidecar reconciled" +logs="$(k logs "deploy/$DEPLOY" -n "$NS" -c stalwart-apply --tail=200 2>/dev/null || true)" +printf '%s\n' "$logs" | grep -E 'catch-all|credentials|reconcile complete|FAIL' | sed 's/^/ /' || true +printf '%s' "$logs" | grep -q 'reconcile complete' || { printf ' no "reconcile complete" in the sidecar log\n' >&2; rollback; } +if printf '%s' "$logs" | grep -q '^FAIL'; then + printf ' sidecar reported FAIL\n' >&2 + rollback +fi + +say "verify: stalwart is serving" +k port-forward -n "$NS" "deploy/$DEPLOY" 18080:8080 >/dev/null 2>&1 & +PF_PID=$! +for _ in $(seq 1 20); do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 http://127.0.0.1:18080/admin/ 2>/dev/null || true)" + case "$code" in 200|301|302|401) break;; *) sleep 2;; esac +done +printf ' GET /admin/ -> %s\n' "${code:-no response}" +case "${code:-}" in 200|301|302|401) ;; *) rollback;; esac + +k port-forward -n "$NS" "deploy/$DEPLOY" 10143:143 >/dev/null 2>&1 & +imap_pid=$! +sleep 3 +if command -v nc >/dev/null 2>&1 && nc -z 127.0.0.1 10143 2>/dev/null; then + printf ' IMAP 143 accepts connections\n' +else + printf ' IMAP 143 check inconclusive (nc unavailable or refused)\n' +fi +kill "$imap_pid" 2>/dev/null || true + +say "done" +printf 'apps-mail is resumed, the sidecar reconciled, and stalwart is serving.\n' +printf 'joris.jonkers and n8n remain unmanaged by design -- Vault holds no password\n' +printf 'for either. Populate secret/platform/mail joris.password and n8n.password,\n' +printf 'then move them into managedAccounts, to bring them under reconciliation.\n'