chore(scripts): collect the cutover and recovery helpers into infra/scripts

infra/scripts/fix-vault-bootstrap-auth.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+# One-time: give vault-bootstrap-auth a Kubernetes auth role so it stops
+# depending on a static token that expires unnoticed.
+#
+# The Job authenticates with a token from the vault-bootstrap-token Secret. That
+# token is invalid (`vault token lookup` -> 403), so the Job has been failing,
+# its pods are reaped, and every auth role and policy added to bootstrap-auth.sh
+# since it expired was applied by Flux and never reached the server. Three
+# backups are blocked on exactly that.
+#
+# This needs a privileged Vault token ONCE. Afterwards the Job logs in with its
+# ServiceAccount and there is nothing left to expire.
+#
+# The companion manifest change is already merged (fleet-infra #128), so the Job
+# currently fails with `invalid role name "vault-bootstrap"` -- this script is
+# what creates that role. It only adds a policy and a role; it changes nothing
+# that is currently working.
+set -euo pipefail
+
+CTX="${KUBE_CONTEXT:-personal}"
+NS=data-system
+SA=vault-bootstrap-auth
+ROLE=vault-bootstrap
+POLICY=vault-bootstrap
+PORT=18200
+PF_PID=""
+
+k() { kubectl --context "$CTX" "$@"; }
+say() { printf '\n== %s\n' "$*"; }
+die() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; }
+cleanup() {
+  [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null || true
+  k delete pod vault-bootstrap-login-probe -n "$NS" --ignore-not-found >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+say "preflight"
+k config current-context >/dev/null || die "context $CTX unreachable"
+printf '  context: %s\n' "$(k config current-context)"
+k get sa "$SA" -n "$NS" >/dev/null 2>&1 || die "ServiceAccount $NS/$SA not found"
+printf '  serviceaccount: %s/%s\n' "$NS" "$SA"
+
+# Vault's TokenReview call needs the SA bound to system:auth-delegator. The Job
+# manifest already installs that binding; check rather than assume.
+if k get clusterrolebinding vault-bootstrap-auth-token-reviewer >/dev/null 2>&1; then
+  printf '  token-reviewer binding: present\n'
+else
+  printf '  token-reviewer binding: MISSING -- kubernetes auth login will fail\n' >&2
+  die "apply cluster/flux/apps/data/vault/bootstrap-auth-job.yaml first"
+fi
+
+say "opening a port-forward to vault"
+k port-forward -n "$NS" svc/vault "$PORT":8200 >/dev/null 2>&1 &
+PF_PID=$!
+export VAULT_ADDR="http://127.0.0.1:${PORT}"
+ready=no
+for _ in $(seq 1 20); do
+  if curl -fsS --max-time 2 "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200" >/dev/null 2>&1; then
+    ready=yes; break
+  fi
+  sleep 2
+done
+[ "$ready" = yes ] || die "vault is not reachable on $VAULT_ADDR"
+
+sealed="$(curl -fsS "${VAULT_ADDR}/v1/sys/health?standbyok=true&sealedcode=200" | sed -n 's/.*"sealed":\([a-z]*\).*/\1/p')"
+printf '  sealed: %s\n' "$sealed"
+[ "$sealed" = "false" ] || die "vault is sealed -- unseal before bootstrapping auth"
+
+say "privileged token"
+printf 'Paste a Vault token allowed to write policies and auth roles.\n'
+printf 'The initial root token works. Input is hidden.\n'
+printf 'token: '
+read -rs VAULT_TOKEN
+printf '\n'
+export VAULT_TOKEN
+[ -n "$VAULT_TOKEN" ] || die "no token provided"
+
+command -v vault >/dev/null 2>&1 \
+  || die "the vault CLI is not installed locally; install it and re-run"
+vault token lookup >/dev/null 2>&1 || die "that token is not valid against $VAULT_ADDR"
+printf '  token accepted\n'
+
+say "writing policy $POLICY"
+# Scoped to what bootstrap-auth.sh actually does: enable the kubernetes auth
+# method and the kvv2/database/rabbitmq engines, write the policies and roles it
+# defines, manage the auth-api transit signing key, and read the engine admin
+# credentials it configures those engines with.
+vault policy write "$POLICY" - <<'HCL'
+path "sys/auth" {
+  capabilities = ["read", "list"]
+}
+path "sys/auth/*" {
+  capabilities = ["create", "update", "read", "sudo"]
+}
+path "sys/mounts" {
+  capabilities = ["read", "list"]
+}
+path "sys/mounts/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "sys/policies/acl/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "auth/kubernetes/config" {
+  capabilities = ["create", "update", "read"]
+}
+path "auth/kubernetes/role/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "database/config/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "database/roles/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "rabbitmq/config/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "rabbitmq/roles/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "transit/keys/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "kvv2/data/*" {
+  capabilities = ["create", "update", "read"]
+}
+path "kvv2/metadata/*" {
+  capabilities = ["create", "update", "read", "list", "delete"]
+}
+# Engine admin credentials the bootstrap reads to configure database/ and
+# rabbitmq/ above. Read-only: the bootstrap never writes these.
+path "secret/data/platform/postgres" {
+  capabilities = ["read"]
+}
+path "secret/data/platform/rabbitmq" {
+  capabilities = ["read"]
+}
+HCL
+printf '  policy written\n'
+
+say "creating kubernetes auth role $ROLE"
+# Bound to this one ServiceAccount in this one namespace. The TTL only has to
+# outlast a single bootstrap run.
+vault write "auth/kubernetes/role/${ROLE}" \
+  bound_service_account_names="$SA" \
+  bound_service_account_namespaces="$NS" \
+  token_policies="$POLICY" \
+  ttl=20m \
+  max_ttl=1h >/dev/null
+printf '  role bound to %s/%s, ttl=20m\n' "$NS" "$SA"
+
+say "verifying the ServiceAccount can log in"
+# The only verification that counts: log in from inside the cluster as that SA,
+# the same way the Job will.
+k delete pod vault-bootstrap-login-probe -n "$NS" --ignore-not-found >/dev/null 2>&1
+cat <<EOF | k apply -f - >/dev/null
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vault-bootstrap-login-probe
+  namespace: ${NS}
+spec:
+  serviceAccountName: ${SA}
+  restartPolicy: Never
+  nodeSelector:
+    platform.jorisjonkers.dev/site: frankfurt
+  containers:
+    - name: probe
+      image: hashicorp/vault:1.21.2
+      env:
+        - name: VAULT_ADDR
+          value: http://vault.data-system.svc.cluster.local:8200
+      command: ['/bin/sh','-ec']
+      args:
+        - |
+          VAULT_TOKEN="\$(vault write -field=token auth/kubernetes/login \\
+            role=${ROLE} \\
+            jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token)"
+          export VAULT_TOKEN
+          echo "login: ok"
+          echo "policies: \$(vault token lookup -format=json | grep -o '"${POLICY}"' | head -1)"
+          vault policy read vso >/dev/null && echo "policy read: ok"
+          vault read -field=bound_service_account_names auth/kubernetes/role/vso >/dev/null && echo "role read: ok"
+EOF
+phase=""
+for _ in $(seq 1 40); do
+  phase="$(k get pod vault-bootstrap-login-probe -n "$NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+  case "$phase" in Succeeded|Failed) break;; esac
+  sleep 3
+done
+printf '  probe phase: %s\n' "$phase"
+k logs vault-bootstrap-login-probe -n "$NS" 2>&1 | sed 's/^/    /'
+[ "$phase" = "Succeeded" ] || die "the ServiceAccount could not log in with role $ROLE"
+
+say "done"
+printf 'Policy %s and kubernetes auth role %s exist, and %s/%s can log in.\n' \
+  "$POLICY" "$ROLE" "$NS" "$SA"
+printf '\nThe manifest change is already live (fleet-infra #128), so the Job only\n'
+printf 'needs a re-run:\n'
+printf '  kubectl --context %s delete job vault-bootstrap-auth -n %s\n' "$CTX" "$NS"
+printf '  flux --context %s reconcile kustomization apps-data --with-source\n' "$CTX"
+printf '\nThen confirm the roles and policies it was never able to write have landed:\n'
+printf '  kubectl --context %s get job vault-bootstrap-auth -n %s\n' "$CTX" "$NS"
+printf '  kubectl --context %s get vaultstaticsecret -n observability\n' "$CTX"
+printf '\nThe vault-bootstrap-token Secret is then unused and can be deleted.\n'

infra/scripts/flip-flux-source-to-monorepo.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+###############################################################################
+# Roll the live Flux source back to ExtraToast/personal-stack @ main.
+#
+# Three steps, not two. kubectl apply does NOT remove spec.provider, so
+# applying the monorepo manifest while provider: github lingers leaves the
+# source rejecting itself with the inverse error:
+#   "secretRef with github app data must be specified when provider is github"
+# That is what happened on the first rollback, so the field is deleted
+# explicitly here.
+#
+# Usage: bash infra/scripts/flip-flux-source-to-monorepo.sh
+#        MONOREPO=/path/to/personal-stack bash flip-flux-source-to-monorepo.sh
+###############################################################################
+set -uo pipefail
+
+CTX="${KUBE_CONTEXT:-personal}"
+# infra/scripts/<this> -> repo root is two levels up. Overridable for a clone
+# checked out elsewhere.
+MONOREPO="${MONOREPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
+EXPECT_URL="https://github.com/ExtraToast/personal-stack"
+SYNC="platform/cluster/flux/clusters/production/flux-system/gotk-sync.yaml"
+
+k() { kubectl --context "$CTX" "$@"; }
+die() { echo "ABORT: $*" >&2; exit 1; }
+
+echo "== 1. verify the monorepo manifest =="
+[ -f "$MONOREPO/$SYNC" ] || die "$MONOREPO/$SYNC not found (set MONOREPO=...)"
+grep -q "$EXPECT_URL" "$MONOREPO/$SYNC" || die "$SYNC does not point at $EXPECT_URL"
+grep -q "provider:" "$MONOREPO/$SYNC" && die "$SYNC unexpectedly sets provider; review before rolling back"
+echo "   points at $EXPECT_URL, no provider field"
+
+echo "== 2. drop the GitHub App credentials =="
+# Ignore failures: the keys may already be absent, which is fine.
+k patch secret flux-system -n flux-system --type=json -p '[
+  {"op":"remove","path":"/data/githubAppID"},
+  {"op":"remove","path":"/data/githubAppInstallationID"},
+  {"op":"remove","path":"/data/githubAppPrivateKey"}]' 2>/dev/null \
+  && echo "   removed" || echo "   already absent"
+
+echo "== 3. drop spec.provider from the live GitRepository =="
+k patch gitrepository flux-system -n flux-system --type=json \
+  -p '[{"op":"remove","path":"/spec/provider"}]' 2>/dev/null \
+  && echo "   removed" || echo "   already absent"
+
+echo "== 4. apply the monorepo source =="
+cd "$MONOREPO" || die "cannot cd $MONOREPO"
+k apply -k platform/cluster/flux/clusters/production/flux-system || die "apply failed"
+
+echo "== 5. reconcile =="
+flux --context "$CTX" reconcile source git flux-system -n flux-system --timeout=180s || true
+flux --context "$CTX" reconcile kustomization flux-system -n flux-system --timeout=300s || true
+
+echo "== 6. verify =="
+k get gitrepository -n flux-system flux-system \
+  -o jsonpath='   url={.spec.url}{"\n"}   branch={.spec.ref.branch}{"\n"}   provider={.spec.provider}{"\n"}   ready={.status.conditions[?(@.type=="Ready")].status}{"\n"}   artifact={.status.artifact.revision}{"\n"}'
+echo
+echo "   Reconciling the services most likely to have been mid-roll:"
+for ks in apps-mail apps-utility-system apps-stateless; do
+  flux --context "$CTX" reconcile kustomization "$ks" --timeout=180s >/dev/null 2>&1 \
+    && echo "      $ks reconciled" || echo "      $ks FAILED"
+done
+echo "   pods not Running/Completed:"
+k get pods -A --no-headers | awk '$4!="Running" && $4!="Completed" {print "      "$1"/"$2"  "$4}' | head -15
+echo
+echo "KNOWN: agents-api cannot roll back. The database is at schema 23, created"
+echo "by v0.19.1; the monorepo image knows 20 migrations and exits instead of"
+echo "running against a newer schema. Its healthy v0.19.1 pod keeps serving, but"
+echo "the Deployment's desired state is the broken image."

infra/scripts/flip-flux-source-to-org.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+###############################################################################
+# Flip the live Flux source to JorisJonkers-dev/fleet-infra @ deploy/production.
+#
+# The secret and the GitRepository must change together: source-controller
+# rejects a secret carrying GitHub App keys unless provider is github, and
+# rejects provider github unless the keys are present. Either half alone leaves
+# the source InvalidProviderConfiguration, so this does both back to back.
+#
+# Usage: bash infra/scripts/flip-flux-source-to-org.sh
+# Undo:  bash flip-flux-source-to-monorepo.sh
+###############################################################################
+set -uo pipefail
+
+CTX="${KUBE_CONTEXT:-personal}"
+APP_ID=3942418
+INSTALL_ID=142960823
+EXPECT_URL="https://github.com/JorisJonkers-dev/fleet-infra"
+EXPECT_BRANCH="deploy/production"
+WORK="${WORK:-/tmp/flip-org-$$}"
+
+k() { kubectl --context "$CTX" "$@"; }
+die() { echo "ABORT: $*" >&2; exit 1; }
+
+echo "== 1. checkout deploy/production =="
+rm -rf "$WORK"
+git clone -q --depth 1 --branch "$EXPECT_BRANCH" "$EXPECT_URL" "$WORK" \
+  || die "cannot clone $EXPECT_URL ($EXPECT_BRANCH)"
+cd "$WORK" || die "cannot cd $WORK"
+echo "   $(git log --oneline -1)"
+
+echo "== 2. verify the manifest before applying it =="
+SYNC=cluster/flux/clusters/production/flux-system/gotk-sync.yaml
+[ -f "$SYNC" ] || die "$SYNC missing"
+grep -q "provider: github" "$SYNC" || die "gotk-sync has no 'provider: github'; the App keys would be rejected"
+grep -q "$EXPECT_URL" "$SYNC"     || die "gotk-sync does not point at $EXPECT_URL"
+grep -q "branch: $EXPECT_BRANCH"  "$SYNC" || die "gotk-sync does not target $EXPECT_BRANCH"
+echo "   provider: github, url and branch as expected"
+
+# apps-edge pruning must be off for the flip: 38 public IngressRoutes move to
+# other Kustomizations, nothing orders apps-edge after the ones adopting them,
+# and a prune-first reconcile drops public routing until they catch up.
+PRUNE=$(python3 -c "
+import yaml
+for d in yaml.safe_load_all(open('cluster/flux/clusters/production/kustomizations.yaml')):
+    if d and d.get('kind')=='Kustomization' and d['metadata']['name']=='apps-edge':
+        print(d['spec'].get('prune'))
+" 2>/dev/null)
+if [ "$PRUNE" = "False" ]; then
+  echo "   apps-edge prune: false (route relocation is safe)"
+else
+  echo "   WARNING: apps-edge prune=$PRUNE. Public routes may drop briefly during the flip."
+fi
+
+echo "== 3. add the GitHub App credentials to the flux-system secret =="
+KEY_B64=$(k get secret -n agents-system github-app -o jsonpath='{.data.private-key}' 2>/dev/null)
+[ -n "$KEY_B64" ] || die "cannot read agents-system/github-app private-key"
+ID_B64=$(printf '%s' "$APP_ID" | base64)
+INST_B64=$(printf '%s' "$INSTALL_ID" | base64)
+k patch secret flux-system -n flux-system --type merge \
+  -p "{\"data\":{\"githubAppPrivateKey\":\"$KEY_B64\",\"githubAppID\":\"$ID_B64\",\"githubAppInstallationID\":\"$INST_B64\"}}" \
+  || die "secret patch failed"
+echo "   NOTE: the live source is invalid from here until step 4 applies provider: github."
+
+echo "== 4. apply the new source =="
+k apply -k cluster/flux/clusters/production/flux-system || die "apply failed -- run flip-flux-source-to-monorepo.sh"
+
+echo "== 5. reconcile =="
+flux --context "$CTX" reconcile source git flux-system -n flux-system --timeout=180s || true
+flux --context "$CTX" reconcile kustomization flux-system -n flux-system --timeout=300s || true
+
+echo "== 6. verify =="
+k get gitrepository -n flux-system flux-system \
+  -o jsonpath='   url={.spec.url}{"\n"}   branch={.spec.ref.branch}{"\n"}   provider={.spec.provider}{"\n"}   ready={.status.conditions[?(@.type=="Ready")].status}{"\n"}   artifact={.status.artifact.revision}{"\n"}'
+echo
+echo "   Kustomizations not ready:"
+k get kustomization -n flux-system -o json \
+  | python3 -c 'import json,sys
+for k in json.load(sys.stdin)["items"]:
+    c=[x for x in (k.get("status") or {}).get("conditions") or [] if x["type"]=="Ready"]
+    if c and c[0]["status"]!="True": print("     ",k["metadata"]["name"],"-",c[0].get("reason"))'
+echo "   pods not Running/Completed:"
+k get pods -A --no-headers | awk '$4!="Running" && $4!="Completed" {print "      "$1"/"$2"  "$4}' | head -15
+echo "   PVCs (expect 24):" "$(k get pvc -A --no-headers | wc -l | tr -d ' ')"
+echo
+echo "Done. If this went wrong: bash flip-flux-source-to-monorepo.sh"