diff --git a/.github/workflows/linux-lifecycle.yml b/.github/workflows/linux-lifecycle.yml new file mode 100644 index 0000000..f3ae9c9 --- /dev/null +++ b/.github/workflows/linux-lifecycle.yml @@ -0,0 +1,39 @@ +name: Linux lifecycle + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/linux-lifecycle.yml + - scripts/check-linux-lifecycle.sh + - scripts/check-update-rollback-hosted.mjs + +permissions: + contents: read + +jobs: + lifecycle: + # A hosted VM with real PID-1 systemd: a job container cannot prove service installation. + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "24" + - name: Install rootless runtime prerequisites in the disposable VM + run: | + sudo apt-get update + sudo apt-get install -y podman uidmap slirp4netns fuse-overlayfs + - name: Exercise installation, restart, fixture recovery and uninstall + run: bash scripts/check-linux-lifecycle.sh "$RUNNER_TEMP/lifecycle-evidence" + - name: Upload non-secret lifecycle evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: linux-lifecycle-${{ github.sha }} + path: ${{ runner.temp }}/lifecycle-evidence/ + if-no-files-found: error diff --git a/.github/workflows/linux-reboot.yml b/.github/workflows/linux-reboot.yml new file mode 100644 index 0000000..5322a27 --- /dev/null +++ b/.github/workflows/linux-reboot.yml @@ -0,0 +1,34 @@ +name: Linux guest reboot + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/linux-reboot.yml + - scripts/check-linux-reboot.sh + - scripts/install-systemd.sh + - scripts/uninstall-systemd.sh + +permissions: + contents: read + +jobs: + reboot: + # The hosted runner survives while a separate, disposable Ubuntu guest actually reboots. + # KVM availability is experimental on hosted runners: missing support fails explicitly. + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install, reboot and uninstall in a disposable KVM guest + run: bash scripts/check-linux-reboot.sh + - name: Upload sanitized reboot acceptance evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: linux-guest-reboot-${{ github.sha }} + path: ${{ runner.temp }}/reboot-evidence/reboot.log + if-no-files-found: error diff --git a/FEATURES.md b/FEATURES.md index f2248e1..b24e44d 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -2,6 +2,20 @@ A categorized catalog of what's shipped. Cross-linked to `TEST-PLAN.md` checks. +- **Disposable Linux lifecycle evidence:** the manually dispatchable Linux lifecycle workflow + exercises a fresh dedicated-account systemd install with the full rootless Podman image, + HTTP liveness after restart, encrypted fixture backup/restore and non-destructive uninstall. + A local fixture upstream also proves the real CLI updater restores Git and restarts systemd + after a candidate test failure or readiness failure; engine smoke is explicitly stubbed. + Its script refuses non-hosted or occupied hosts. A separate KVM guest workflow exercises an + actual OS reboot, service autostart and persistent database/container-volume fixtures. + Authenticated engine update smoke and conversation/session acceptance remain separate live + gates. → TEST-PLAN: Disposable Linux lifecycle workflow. +- **Service-account image provisioning** uses the same explicit environment as the systemd + daemon so operator XDG/container storage settings cannot redirect a fresh build into another + user's private Podman store, and runs from the service-owned checkout so an operator-private + invocation directory cannot prevent Podman namespace setup. + - Development acceptance policy: behavior changes include reproducible Claude and Codex acceptance definitions and clearly separate automated evidence from live operator validation. Public contributors do not need access to a private QA service. → TEST-PLAN: Development acceptance policy. diff --git a/TEST-PLAN.md b/TEST-PLAN.md index c5226ee..69625f6 100644 --- a/TEST-PLAN.md +++ b/TEST-PLAN.md @@ -3,6 +3,63 @@ Cumulative functional + security regression. Extended per slice. Run top-to-bottom for a full pass. Many checks are manual (require a real Slack workspace + an authenticated `claude` CLI). +## Disposable Linux lifecycle workflow + +- Automated setup: dispatch `.github/workflows/linux-lifecycle.yml` for the candidate ref (a PR + changing this workflow/script also runs it). GitHub-hosted Ubuntu 24.04, Node 24, real PID-1 + systemd and rootless Podman; no job container, provider credentials or chat connection. + `scripts/check-linux-lifecycle.sh` refuses non-hosted runners, occupied fixture paths, service + units and accounts before mutation. It operates only on `/opt/channelgate-lifecycle` and the + newly installed `/var/lib/channelgate-lifecycle` service identity; never an operator deployment. + The installer image build must succeed even when the invoking runner has its own container + storage configuration; the service account must use its own HOME/store and explicit environment. + Invoke the installer by absolute path while cwd is an operator-private directory; the image + probe/build must run from the service-owned checkout rather than inherit that inaccessible cwd. +- Pass evidence: `linux-lifecycle-` artifact records source revision, VM image, versions and + every PASS line. Require successful fresh install/image build, non-root container with zero + effective capabilities/no-new-privileges, enabled active service, distinct healthy instance ID + after real systemd restart, encrypted snapshot while the fixture daemon runs, restored + `before-backup` SQL/config markers, SQLite integrity `ok`, removal of stale WAL/SHM and stray + config, healthy restart and uninstall preserving the account/database/encrypted backup. + These checks are engine-independent because they issue no engine turn. Update transaction + unit evidence is uploaded separately as `update-fixture-tests.tap`; injected failures do not + count as real authenticated engine update/rollback acceptance. +- Hosted updater operations: `scripts/check-update-rollback-hosted.mjs` is restricted to that + exact disposable checkout/runtime, controls the LOCAL fixture's engine smoke and test commands, + and creates a local bare Git upstream. It invokes the unmodified CLI updater first against a + candidate whose test exits 42, then one whose live health reports an incorrect revision. + Require two durable `rolled_back` results, checkout/running revision A, new healthy service + instance, released update lock, private operator recovery snapshot and unchanged SQL marker. + The readiness failure must name the deliberately wrong live revision, not just any timeout; + both recovery-snapshot SQL/config markers must match and snapshot integrity must be `ok`. + Finally restore the original tested source revision and verify its fresh healthy instance. + Public evidence must name the smoke stub; no Claude/Codex authentication is proven here. +- Guest OS reboot: dispatch `.github/workflows/linux-reboot.yml` for the candidate ref. The + hosted runner must expose KVM; unavailable acceleration fails explicitly. The wrapper verifies + the official Ubuntu Noble image checksum, boots a separate cloud-init guest, installs Node 24 + and the real service/full image, writes `persisted-through-os-reboot` into SQLite and a rootless + named volume, and reboots the GUEST while the runner stays alive. Pass only with different OS + boot ID and daemon instance ID, enabled active service without manual post-boot start, usable + rootless runtime, intact SQL/volume markers, SQLite integrity `ok` and uninstall preserving data. + Artifact `linux-guest-reboot-` contains sanitized logs only. No keys, disk images, runtime + config or databases are uploaded. This operations case is engine-independent; it does not prove + a resumed Claude/Codex conversation. +- [ ] Post-reboot conversation gate (both configured engines): on a separate disposable Linux VM, install + the candidate and create a channel that writes `LIFECYCLE-BEFORE-REBOOT` in its own work folder. + Record instance ID and engine/session identity, reboot the machine, then ask each engine in + its existing thread to read the marker. Require automatic service start without manual repair, + a new daemon instance ID, preserved marker/session and container-only engine execution. + Engine-free guest reboot evidence cannot clear this conversation/session gate. +- [ ] Authenticated update rollback gate (Claude and Codex): on that disposable deployment, + configure both engine credentials in its own service identity and a local fixture upstream. + Baseline revision A must answer the fixed update smoke response for both engines. Create + fast-forward candidate B with an intentional failing test; invoke `npm run update`. + Require a visible candidate failure, durable `rolled_back` state, restored A checkout and + an operator recovery snapshot of database/config (runtime data is not automatically rolled + back), a new healthy A instance, passing smoke for both baseline engines and a new + ordinary turn in each existing channel. Repeat with a candidate that passes tests but fails + readiness after restart. Never perform induced-failure checks on a production deployment. + ## Base modes and independent options (2026-09-08) - Automated: `modes`, `channel-settings-modal`, `mode-command-audit`, `folders-settings`, diff --git a/scripts/check-linux-lifecycle.sh b/scripts/check-linux-lifecycle.sh new file mode 100644 index 0000000..3d4f84e --- /dev/null +++ b/scripts/check-linux-lifecycle.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Destructive operations acceptance, ONLY on an empty GitHub-hosted Ubuntu VM. +# No provider credentials, chat connections or engine prompts are needed or accepted here. +set -euo pipefail +if [ "${GITHUB_ACTIONS:-}" != true ] || [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ] || + [ "$(cat /proc/1/comm)" != systemd ]; then + echo "Refusing: this check requires a disposable GitHub-hosted VM with PID-1 systemd." >&2 + exit 2 +fi +SOURCE_DIR="$(cd "$(dirname "$0")/.." && pwd)" +EVIDENCE="${1:?Pass the evidence output directory}" +APP_DIR=/opt/channelgate-lifecycle +SERVICE_USER=cg-lifecycle +SERVICE_HOME=/var/lib/channelgate-lifecycle +for occupied in "$APP_DIR" "$SERVICE_HOME" /etc/systemd/system/channelgate.service /etc/systemd/system/claude-gateway.service; do + [ ! -e "$occupied" ] || { echo "Refusing occupied lifecycle fixture: $occupied" >&2; exit 2; } +done +if id "$SERVICE_USER" >/dev/null 2>&1 || id claude-gateway >/dev/null 2>&1; then + echo "Refusing existing lifecycle or legacy service account" >&2; exit 2 +fi +mkdir -p "$EVIDENCE" +exec > >(tee "$EVIDENCE/lifecycle.log") 2>&1 +printf 'revision=%s\n' "$(git -C "$SOURCE_DIR" rev-parse HEAD)" +printf 'runner=%s\n' "${ImageOS:-unknown} ${ImageVersion:-unknown}" +uname -sr +node --version +podman --version +systemctl --version | head -1 +df -h /opt /var/lib + +cleanup() { + outcome=$? + trap - EXIT + if [ -f /etc/systemd/system/channelgate.service ]; then + sudo bash "$APP_DIR/scripts/uninstall-systemd.sh" --system || true + fi + printf 'exit_code=%s\n' "$outcome" + exit "$outcome" +} +trap cleanup EXIT +pass() { printf 'PASS %s\n' "$1"; } +as_service() { + sudo runuser -u "$SERVICE_USER" -- env -i HOME="$SERVICE_HOME" CHANNELGATE_DIR="$SERVICE_HOME" \ + CHANNELGATE_DB="" CLAUDE_GATEWAY_DIR="" CLAUDE_GATEWAY_DB="" \ + XDG_RUNTIME_DIR="/run/user/$(id -u "$SERVICE_USER")" \ + DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$(id -u "$SERVICE_USER")/bus" \ + PATH="$PATH" /bin/sh -c 'cd "$HOME" && exec "$@"' sh "$@" +} +health() { + node --input-type=module - "$1" <<'NODE' +import assert from 'node:assert/strict'; +const previous = process.argv[2]; +for (let attempt = 0; attempt < 90; attempt += 1) { + try { + const response = await fetch('http://127.0.0.1:4747/api/health', { signal: AbortSignal.timeout(2000) }); + assert.equal(response.status, 200); + const data = await response.json(); + assert.equal(data.ok, true); + assert.ok(data.instanceId && data.instanceId !== previous); + assert.equal(data.slack.connected, false); + console.log(data.instanceId); + process.exit(0); + } catch { await new Promise((resolve) => setTimeout(resolve, 1000)); } +} +throw new Error('Fresh healthy daemon instance did not become available'); +NODE +} + +# Clone public source only; never copy the caller's home, credentials or runtime state. +sudo git clone --no-checkout "https://github.com/${GITHUB_REPOSITORY:?}.git" "$APP_DIR" +sudo git -C "$APP_DIR" fetch origin "$(git -C "$SOURCE_DIR" rev-parse HEAD)" +sudo git -C "$APP_DIR" checkout --detach "$(git -C "$SOURCE_DIR" rev-parse HEAD)" +BOOTSTRAP_ROOT="$(mktemp -d)" +sudo env PATH="$PATH" CHANNELGATE_DIR="$BOOTSTRAP_ROOT" CHANNELGATE_DB="" CLAUDE_GATEWAY_DIR="" CLAUDE_GATEWAY_DB="" \ + bash "$APP_DIR/scripts/install.sh" --without-whisper +# Public, disposable fixture password prevents a generated bootstrap secret reaching CI logs. +# The daemon listens only on loopback; no provider/chat tokens are configured. +sudo tee "$APP_DIR/.env" >/dev/null <<'ENV' +ADMIN_PASSWORD=public-disposable-lifecycle-fixture +CG_BIND_HOST=127.0.0.1 +PORT=4747 +ENV +sudo env PATH="$PATH" CG_SERVICE_USER="$SERVICE_USER" CG_SERVICE_HOME="$SERVICE_HOME" \ + bash "$APP_DIR/scripts/install-systemd.sh" +FIRST_INSTANCE="$(health '')" +systemctl is-enabled channelgate.service +systemctl is-active channelgate.service +[ "$(systemctl show channelgate.service -p User --value)" = "$SERVICE_USER" ] +[ "$(systemctl show channelgate.service -p Delegate --value)" = yes ] +[ "$(sudo stat -c %a "$SERVICE_HOME/service.env")" = 600 ] +as_service podman info --format '{{.Host.Security.Rootless}}' | grep -qx true +pass 'fresh systemd installation, dedicated identity, enablement and HTTP liveness' +as_service podman run --rm --userns=keep-id --cap-drop=all --security-opt=no-new-privileges \ + --network=bridge channelgate/runtime:latest node --input-type=module -e \ + 'import assert from "node:assert/strict"; import {readFileSync} from "node:fs"; assert.notEqual(process.getuid(),0); const s=readFileSync("/proc/self/status","utf8"); assert.match(s,/CapEff:\s+0+\n/); assert.match(s,/NoNewPrivs:\s+1\n/); console.log("container uid="+process.getuid()+" zero capabilities, no-new-privileges");' +pass 'production image runs with rootless identity and dropped capabilities (no engine prompt)' +sudo systemctl restart channelgate.service +SECOND_INSTANCE="$(health "$FIRST_INSTANCE")" +pass 'real systemd restart produces a new healthy daemon instance' + +# Use the freshly installed disposable daemon database, never an operator deployment. +sudo systemctl stop channelgate.service +as_service node --input-type=module - "$SERVICE_HOME" <<'NODE' +import { DatabaseSync } from 'node:sqlite'; +import { writeFileSync } from 'node:fs'; +const root = process.argv[2]; +const db = new DatabaseSync(`${root}/gateway.db`); +db.exec("CREATE TABLE lifecycle_proof(value TEXT); INSERT INTO lifecycle_proof VALUES ('before-backup')"); +db.close(); +writeFileSync(`${root}/config/lifecycle-proof.json`, '{"value":"before-backup"}\n'); +NODE +sudo systemctl start channelgate.service +THIRD_INSTANCE="$(health "$SECOND_INSTANCE")" +as_service env CG_BACKUP_PASSPHRASE=public-disposable-backup-fixture bash "$APP_DIR/scripts/backup-config.sh" +as_service env CG_BACKUP_PASSPHRASE=public-disposable-backup-fixture bash "$APP_DIR/scripts/restore-drill.sh" +sudo systemctl stop channelgate.service +as_service node --input-type=module - "$SERVICE_HOME" <<'NODE' +import { DatabaseSync } from 'node:sqlite'; +import { writeFileSync } from 'node:fs'; +const root = process.argv[2]; +const db = new DatabaseSync(`${root}/gateway.db`); +db.exec("UPDATE lifecycle_proof SET value='after-backup'"); +db.close(); +writeFileSync(`${root}/config/stray-lifecycle.json`, '{}'); +writeFileSync(`${root}/gateway.db-wal`, 'stale-wal'); +writeFileSync(`${root}/gateway.db-shm`, 'stale-shm'); +NODE +as_service env CG_BACKUP_PASSPHRASE=public-disposable-backup-fixture CG_RESTORE_CONFIRM=YES \ + bash "$APP_DIR/scripts/restore-config.sh" +as_service node --input-type=module - "$SERVICE_HOME" <<'NODE' +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +import { existsSync, readFileSync } from 'node:fs'; +const root = process.argv[2]; +assert.equal(existsSync(`${root}/gateway.db-wal`), false); +assert.equal(existsSync(`${root}/gateway.db-shm`), false); +assert.equal(existsSync(`${root}/config/stray-lifecycle.json`), false); +assert.equal(JSON.parse(readFileSync(`${root}/config/lifecycle-proof.json`)).value, 'before-backup'); +const db = new DatabaseSync(`${root}/gateway.db`, { readOnly: true }); +assert.equal(db.prepare('SELECT value FROM lifecycle_proof').get().value, 'before-backup'); +assert.equal(db.prepare('PRAGMA integrity_check').get().integrity_check, 'ok'); +db.close(); +NODE +sudo systemctl start channelgate.service +health "$THIRD_INSTANCE" +pass 'live encrypted fixture snapshot, disposable drill, replacement restore and healthy restart' + +# Existing injected transaction tests are separately labelled: these do not claim a real update +# across authenticated engine versions or an induced failure in a production deployment. +as_service env -u CHANNELGATE_DIR -u CHANNELGATE_DB -u CLAUDE_GATEWAY_DIR -u CLAUDE_GATEWAY_DB \ + -u CG_WORKSPACE_DIR -u CG_TEST_SCRATCH \ + bash -c 'cd "$1" && node --test --test-reporter=tap test/update-runner.test.js test/update-state.test.js test/update-smoke.test.js' bash "$APP_DIR" \ + > "$EVIDENCE/update-fixture-tests.tap" 2>&1 +pass 'injected update failure/rollback and container smoke regression tests' +as_service env CG_DISPOSABLE_LIFECYCLE=1 node "$APP_DIR/scripts/check-update-rollback-hosted.mjs" +sudo bash "$APP_DIR/scripts/uninstall-systemd.sh" --system +[ ! -e /etc/systemd/system/channelgate.service ] +if systemctl is-active --quiet channelgate.service; then echo 'Service remained active after uninstall'; exit 1; fi +id "$SERVICE_USER" >/dev/null +as_service test -s "$SERVICE_HOME/gateway.db" +as_service test -s "$SERVICE_HOME/backups/config.tar.gz.enc" +pass 'uninstall removes and stops system unit while preserving account, database and backup' +printf '%s\n' \ + 'NOT RUN: actual host reboot and post-reboot recovery (hosted job does not survive reboot).' \ + 'NOT RUN: authenticated Claude/Codex update smoke (real Git/service rollback uses a controlled smoke fixture).' \ + 'NOT RUN: Slack/Airtable/Composio or other external acceptance campaigns.' diff --git a/scripts/check-linux-reboot.sh b/scripts/check-linux-reboot.sh new file mode 100644 index 0000000..31e2989 --- /dev/null +++ b/scripts/check-linux-reboot.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# Real OS reboot acceptance in a disposable nested Ubuntu guest; NEVER reboot the runner. +# Guest setup follows the official Ubuntu/cloud-init QEMU guidance: +# https://documentation.ubuntu.com/public-images/public-images-how-to/launch-qcow-with-qemu/ +# https://docs.cloud-init.io/en/24.3/tutorial/qemu.html +# Requires KVM. Nested virtualization on GitHub-hosted runners is experimental; unavailable +# acceleration is an explicit failure, never a skipped/passing acceptance or a slow TCG fallback. +set -euo pipefail +if [ "${GITHUB_ACTIONS:-}" != true ] || [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ] || + [ "$(cat /proc/1/comm)" != systemd ]; then + echo 'Refusing: requires a disposable GitHub-hosted VM with PID-1 systemd.' >&2 + exit 2 +fi +: "${RUNNER_TEMP:?}" "${GITHUB_REPOSITORY:?}" +[[ "$GITHUB_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || exit 2 +SOURCE_DIR="$(cd "$(dirname "$0")/.." && pwd)" +REVISION="$(git -C "$SOURCE_DIR" rev-parse HEAD)" +[[ "$REVISION" =~ ^[a-f0-9]{40}$ ]] || exit 2 +EVIDENCE="$RUNNER_TEMP/reboot-evidence" +FIXTURE="$RUNNER_TEMP/channelgate-reboot-guest" +[ ! -e "$FIXTURE" ] || { echo 'Refusing occupied reboot fixture.' >&2; exit 2; } +umask 077 +mkdir "$FIXTURE" +mkdir -p "$EVIDENCE" +exec > >(tee "$EVIDENCE/reboot.log") 2>&1 +QEMU_PID='' +cleanup() { + outcome=$? + trap - EXIT + if [ -n "$QEMU_PID" ]; then + kill "$QEMU_PID" 2>/dev/null || true + wait "$QEMU_PID" 2>/dev/null || true + fi + # No disks, cloud-init data, private keys, runtime files or raw guest logs are retained. + rm -rf -- "$FIXTURE" + printf 'exit_code=%s\n' "$outcome" + exit "$outcome" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +printf 'revision=%s\nrunner=%s %s\n' "$REVISION" "${ImageOS:-unknown}" "${ImageVersion:-unknown}" +if [ ! -c /dev/kvm ]; then + echo 'UNAVAILABLE: /dev/kvm is absent; actual guest reboot acceptance was NOT RUN.' + exit 1 +fi +sudo apt-get update -qq > "$FIXTURE/host-apt.log" 2>&1 +sudo apt-get install -y -qq qemu-system-x86 qemu-utils cloud-image-utils acl >> "$FIXTURE/host-apt.log" 2>&1 +sudo setfacl -m "u:$(id -u):rw" /dev/kvm +[ -r /dev/kvm ] && [ -w /dev/kvm ] || { + echo 'UNAVAILABLE: KVM is inaccessible; actual guest reboot acceptance was NOT RUN.'; exit 1; +} +# Sparse backing disk still needs real space for the full production toolchain build. +[ "$(df -Pk "$FIXTURE" | awk 'NR==2 {print $4}')" -ge 25165824 ] || { + echo 'UNAVAILABLE: fewer than 24 GiB free for the production-image guest fixture.'; exit 1; +} +echo 'CHECK downloading official Ubuntu Noble cloud image and checksum manifest' +IMAGE=noble-server-cloudimg-amd64.img +IMAGE_BASE=https://cloud-images.ubuntu.com/noble/current +curl --fail --silent --show-error --location --retry 3 "$IMAGE_BASE/SHA256SUMS" -o "$FIXTURE/SHA256SUMS" +curl --fail --silent --show-error --location --retry 3 "$IMAGE_BASE/$IMAGE" -o "$FIXTURE/$IMAGE" +( + cd "$FIXTURE" + awk -v image="$IMAGE" '$2 == image || $2 == "*" image { print }' SHA256SUMS > image.sha256 + [ "$(wc -l < image.sha256)" -eq 1 ] + sha256sum --check image.sha256 +) +printf 'image_sha256=%s\n' "$(awk '{print $1}' "$FIXTURE/image.sha256")" +qemu-img create -q -f qcow2 -F qcow2 -b "$FIXTURE/$IMAGE" "$FIXTURE/guest.qcow2" 48G +ssh-keygen -q -t ed25519 -N '' -C disposable-reboot-fixture -f "$FIXTURE/ssh-key" +cat > "$FIXTURE/user-data" < "$FIXTURE/meta-data" +cloud-localds "$FIXTURE/seed.img" "$FIXTURE/user-data" "$FIXTURE/meta-data" +# Run as the runner user. The owned child lives only for this script and is reaped by its trap. +qemu-system-x86_64 -enable-kvm -cpu host -smp 2 -m 4096 -display none -monitor none \ + -serial "file:$FIXTURE/serial.log" \ + -drive "file=$FIXTURE/guest.qcow2,format=qcow2,if=virtio" \ + -drive "file=$FIXTURE/seed.img,format=raw,if=virtio" \ + -netdev user,id=net0,ipv6=off,hostfwd=tcp:127.0.0.1:2222-:22 -device virtio-net-pci,netdev=net0 \ + > "$FIXTURE/qemu.log" 2>&1 & +QEMU_PID=$! +SSH=(ssh -i "$FIXTURE/ssh-key" -p 2222 -o BatchMode=yes -o ConnectTimeout=5 + -o StrictHostKeyChecking=accept-new -o "UserKnownHostsFile=$FIXTURE/known_hosts" + -o ServerAliveInterval=15 -o ServerAliveCountMax=4 reboot-check@127.0.0.1) +wait_ssh() { + for attempt in $(seq 1 120); do + kill -0 "$QEMU_PID" 2>/dev/null || { + echo 'UNAVAILABLE: KVM guest exited before SSH readiness; reboot acceptance incomplete.' + tail -n 20 "$FIXTURE/qemu.log" + return 1; + } + if "${SSH[@]}" true >/dev/null 2>&1; then return 0; fi + sleep 3 + done + echo 'FAIL guest SSH readiness timed out'; return 1 +} +wait_ssh +"${SSH[@]}" sudo cloud-init status --wait > "$FIXTURE/cloud-init.log" 2>&1 +echo 'PASS disposable KVM Ubuntu guest booted' +# Arguments are restricted to an owner/repo slug and a full SHA before SSH constructs a command. +"${SSH[@]}" sudo bash -s -- "$GITHUB_REPOSITORY" "$REVISION" <<'GUEST' +set -euo pipefail +[ "$(cat /proc/1/comm)" = systemd ] +[ "$(hostname)" = channelgate-reboot-fixture ] +APP_DIR=/opt/channelgate-reboot +SERVICE_USER=cg-reboot +SERVICE_HOME=/var/lib/channelgate-reboot +for occupied in "$APP_DIR" "$SERVICE_HOME" /etc/systemd/system/channelgate.service /etc/systemd/system/claude-gateway.service; do + [ ! -e "$occupied" ] || { echo 'FAIL occupied guest fixture'; exit 2; } +done +if id "$SERVICE_USER" >/dev/null 2>&1 || id claude-gateway >/dev/null 2>&1; then exit 2; fi +phase=prerequisites +setup_failed() { + outcome=$? + printf 'FAIL guest setup phase=%s line=%s\n' "$phase" "$1" + # Only synthetic public-source installation output, bounded and scrubbed. Never print the + # daemon journal, environment/config, cloud-init seed, SSH key or runtime database. + for log in /var/tmp/reboot-prerequisites.log /var/tmp/reboot-install.log; do + if [ -f "$log" ]; then + tail -n 60 "$log" | sed -E 's/public-disposable-[A-Za-z0-9-]+/[fixture-password]/g; s/(sk-|xox[baprs]-)[A-Za-z0-9_-]+/[redacted]/g' + fi + done + exit "$outcome" +} +trap 'setup_failed "$LINENO"' ERR +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq > /var/tmp/reboot-prerequisites.log 2>&1 +apt-get install -y -qq ca-certificates curl git xz-utils podman uidmap slirp4netns fuse-overlayfs >> /var/tmp/reboot-prerequisites.log 2>&1 +phase=node +mkdir /var/tmp/reboot-node +cd /var/tmp/reboot-node +curl -fsSL --retry 3 https://nodejs.org/dist/latest-v24.x/SHASUMS256.txt -o SHASUMS256.txt +awk '$2 ~ /^node-v24\.[0-9]+\.[0-9]+-linux-x64\.tar\.xz$/ {print}' SHASUMS256.txt > node.sha256 +[ "$(wc -l < node.sha256)" -eq 1 ] +NODE_ARCHIVE="$(awk '{print $2}' node.sha256)" +curl -fsSL --retry 3 "https://nodejs.org/dist/latest-v24.x/$NODE_ARCHIVE" -o "$NODE_ARCHIVE" +sha256sum --check node.sha256 +tar -xJf "$NODE_ARCHIVE" -C /usr/local --strip-components=1 +printf 'node_version=%s\nnode_sha256=%s\n' "$(node --version)" "$(awk '{print $1}' node.sha256)" +phase=public-source +echo 'CHECK public browser CDN connectivity from the IPv4 guest network' +curl -4 --head --location --silent --show-error --connect-timeout 15 --max-time 45 \ + --output /dev/null --write-out 'browser_cdn_http=%{http_code} connect_seconds=%{time_connect}\n' \ + https://cdn.playwright.dev/ || true +git clone --quiet --no-checkout "https://github.com/$1.git" "$APP_DIR" +git -C "$APP_DIR" fetch --quiet origin "$2" +git -C "$APP_DIR" checkout --quiet --detach "$2" +[ "$(git -C "$APP_DIR" rev-parse HEAD)" = "$2" ] +# Public synthetic password avoids first-boot random secrets. No Slack/provider credentials. +cat > "$APP_DIR/.env" <<'ENV' +ADMIN_PASSWORD=public-disposable-reboot-fixture +CG_BIND_HOST=127.0.0.1 +PORT=4747 +ENV +phase=install-dependencies +echo 'CHECK installing daemon dependencies in guest' +CHANNELGATE_DIR=/var/tmp/reboot-bootstrap bash "$APP_DIR/scripts/install.sh" --without-whisper > /var/tmp/reboot-install.log 2>&1 +phase=install-system-service +echo 'CHECK production systemd installer is building the complete rootless image in guest' +CG_SERVICE_USER="$SERVICE_USER" CG_SERVICE_HOME="$SERVICE_HOME" bash "$APP_DIR/scripts/install-systemd.sh" >> /var/tmp/reboot-install.log 2>&1 +cat > /var/tmp/reboot-health.mjs <<'NODE' +import assert from 'node:assert/strict'; +for (let attempt = 0; attempt < 120; attempt++) { + try { + const response = await fetch('http://127.0.0.1:4747/api/health', { signal: AbortSignal.timeout(2000) }); + assert.equal(response.status, 200); + const data = await response.json(); + assert.equal(data.ok, true); + assert.equal(data.slack.connected, false); + assert.ok(data.instanceId && data.instanceId !== process.argv[2]); + console.log(data.instanceId); + process.exit(0); + } catch { await new Promise(resolve => setTimeout(resolve, 1000)); } +} +throw Error('Fresh healthy daemon instance did not become available'); +NODE +cat > /var/tmp/reboot-as-service <<'SERVICE' +#!/usr/bin/env bash +set -euo pipefail +uid="$(id -u cg-reboot)" +cd /var/lib/channelgate-reboot +exec runuser -u cg-reboot -- env -i HOME=/var/lib/channelgate-reboot CHANNELGATE_DIR=/var/lib/channelgate-reboot \ + XDG_RUNTIME_DIR="/run/user/$uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$uid/bus" \ + PATH=/usr/local/bin:/usr/bin:/bin "$@" +SERVICE +chmod 700 /var/tmp/reboot-as-service +phase=pre-reboot-state +node /var/tmp/reboot-health.mjs > /var/tmp/reboot-first-instance +systemctl is-enabled --quiet channelgate.service +systemctl is-active --quiet channelgate.service +[ "$(systemctl show channelgate.service -p User --value)" = "$SERVICE_USER" ] +/var/tmp/reboot-as-service podman info --format '{{.Host.Security.Rootless}}' | grep -qx true +/var/tmp/reboot-as-service node --input-type=module <<'NODE' +import { DatabaseSync } from 'node:sqlite'; +const db = new DatabaseSync('/var/lib/channelgate-reboot/gateway.db'); +db.exec("CREATE TABLE reboot_proof(value TEXT); INSERT INTO reboot_proof VALUES ('persisted-through-os-reboot')"); +db.close(); +NODE +/var/tmp/reboot-as-service podman volume create reboot-proof >/dev/null +/var/tmp/reboot-as-service podman run --rm --userns=keep-id --cap-drop=all --security-opt=no-new-privileges \ + --network=none -v reboot-proof:/proof:U channelgate/runtime:latest node -e \ + 'require("node:assert/strict").notEqual(process.getuid(), 0); require("node:fs").writeFileSync("/proof/marker", "persisted-through-os-reboot")' +cat /proc/sys/kernel/random/boot_id > /var/tmp/reboot-first-boot-id +sync +echo 'PASS real service installation, HTTP health, database and rootless volume fixture before reboot' +GUEST +FIRST_BOOT="$("${SSH[@]}" cat /proc/sys/kernel/random/boot_id)" +printf 'guest_boot_before=%s\n' "$FIRST_BOOT" +echo 'CHECK rebooting guest OS; GitHub runner stays alive' +# systemctl schedules a real guest OS reboot. It may close SSH before reporting success. +"${SSH[@]}" sudo systemctl reboot || true +REBOOTED=false +for attempt in $(seq 1 120); do + kill -0 "$QEMU_PID" 2>/dev/null || { echo 'FAIL guest process exited during reboot'; exit 1; } + CURRENT_BOOT="$("${SSH[@]}" cat /proc/sys/kernel/random/boot_id 2>/dev/null || true)" + if [[ "$CURRENT_BOOT" =~ ^[a-f0-9-]{36}$ ]] && [ "$CURRENT_BOOT" != "$FIRST_BOOT" ]; then + REBOOTED=true; break + fi + sleep 3 +done +[ "$REBOOTED" = true ] || { echo 'FAIL guest OS boot ID did not change'; exit 1; } +printf 'guest_boot_after=%s\n' "$CURRENT_BOOT" +"${SSH[@]}" sudo bash -s <<'VERIFY' +set -euo pipefail +trap 'printf "FAIL post-reboot verification line=%s\n" "$LINENO"' ERR +[ "$(cat /proc/sys/kernel/random/boot_id)" != "$(cat /var/tmp/reboot-first-boot-id)" ] +# Do not start/restart the service here: HTTP must return after automatic boot startup. +NEW_INSTANCE="$(node /var/tmp/reboot-health.mjs "$(cat /var/tmp/reboot-first-instance)")" +[ "$NEW_INSTANCE" != "$(cat /var/tmp/reboot-first-instance)" ] +systemctl is-enabled --quiet channelgate.service +systemctl is-active --quiet channelgate.service +[ "$(systemctl show channelgate.service -p User --value)" = cg-reboot ] +[ "$(loginctl show-user cg-reboot -p Linger --value)" = yes ] +/var/tmp/reboot-as-service podman info --format '{{.Host.Security.Rootless}}' | grep -qx true +printf 'daemon_instance_before=%s\ndaemon_instance_after=%s\n' "$(cat /var/tmp/reboot-first-instance)" "$NEW_INSTANCE" +echo 'PASS actual OS reboot, enabled active service autostart, new HTTP instance and rootless runtime' +/var/tmp/reboot-as-service podman run --rm --userns=keep-id --cap-drop=all --security-opt=no-new-privileges \ + --network=none -v reboot-proof:/proof channelgate/runtime:latest node -e \ + 'const a=require("node:assert/strict"); a.notEqual(process.getuid(),0); a.equal(require("node:fs").readFileSync("/proof/marker","utf8"),"persisted-through-os-reboot")' +cat > /var/tmp/reboot-db-check.mjs <<'NODE' +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +const db = new DatabaseSync('/var/lib/channelgate-reboot/gateway.db', { readOnly: true }); +assert.equal(db.prepare('SELECT value FROM reboot_proof').get().value, 'persisted-through-os-reboot'); +assert.equal(db.prepare('PRAGMA integrity_check').get().integrity_check, 'ok'); +db.close(); +NODE +chmod 644 /var/tmp/reboot-db-check.mjs +/var/tmp/reboot-as-service node /var/tmp/reboot-db-check.mjs +echo 'PASS database integrity and rootless named-volume marker survived guest OS reboot' +bash /opt/channelgate-reboot/scripts/uninstall-systemd.sh --system +[ ! -e /etc/systemd/system/channelgate.service ] +if systemctl is-active --quiet channelgate.service; then echo 'FAIL service still active after uninstall'; exit 1; fi +id cg-reboot >/dev/null +/var/tmp/reboot-as-service node /var/tmp/reboot-db-check.mjs +echo 'PASS actual uninstall stops/removes service and preserves service identity and database marker' +VERIFY +echo 'PASS completed real guest OS reboot acceptance; no authenticated engines or external chat services used' diff --git a/scripts/check-update-rollback-hosted.mjs b/scripts/check-update-rollback-hosted.mjs new file mode 100644 index 0000000..ab75a54 --- /dev/null +++ b/scripts/check-update-rollback-hosted.mjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +// Real CLI updater + Git + systemd rollback in the lifecycle VM. Engine smoke and npm test +// commands are controlled in LOCAL fixture commits; this is not authenticated engine acceptance. +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const root = "/var/lib/channelgate-lifecycle"; +assert.equal(repo, "/opt/channelgate-lifecycle", "only the disposable lifecycle checkout is permitted"); +assert.equal(process.env.CHANNELGATE_DIR, root); +assert.equal(process.env.CG_DISPOSABLE_LIFECYCLE, "1"); +assert.notEqual(process.getuid(), 0); +assert.equal(statSync(root).uid, process.getuid()); +assert.equal(JSON.parse(readFileSync(`${root}/config/lifecycle-proof.json`)).value, "before-backup"); +const run = (command, args) => execFileSync(command, args, { cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); +const git = (...args) => run("git", args); +const original = git("rev-parse", "HEAD"); +const upstream = `${root}/update-fixture.git`; +assert.equal(existsSync(upstream), false); +git("config", "user.name", "Lifecycle Fixture"); +git("config", "user.email", "fixture@example.invalid"); +const appFile = `${repo}/src/web/app.js`; +const packageFile = `${repo}/package.json`; +const settingsFile = `${root}/config/settings.json`; +const settings = existsSync(settingsFile) ? JSON.parse(readFileSync(settingsFile)) : {}; +writeFileSync(settingsFile, `${JSON.stringify({ ...settings, whisperEnabled: false, driveSyncEnabled: false }, null, 2)}\n`); +const appSource = readFileSync(appFile, "utf8"); +assert.equal(appSource.split("updateSmoke = runUpdateSmoke,").length, 2); +writeFileSync(appFile, appSource.replace("updateSmoke = runUpdateSmoke,", + 'updateSmoke = async () => ({ ok: true, engines: [{ engine: "fixture-only", ok: true }] }),')); +const manifest = JSON.parse(readFileSync(packageFile)); +manifest.scripts.test = 'node -e "console.log(\'controlled fixture suite passed\')"'; +delete manifest.scripts.pretest; +writeFileSync(packageFile, `${JSON.stringify(manifest, null, 2)}\n`); +git("switch", "-c", "lifecycle-update-fixture"); +git("add", "src/web/app.js", "package.json"); +git("commit", "-m", "fixture: deterministic unauthenticated smoke for updater operations"); +const baseline = git("rev-parse", "HEAD"); +git("clone", "--bare", repo, upstream); +git("remote", "set-url", "origin", upstream); +git("fetch", "origin"); +git("branch", "--set-upstream-to=origin/lifecycle-update-fixture"); + +async function health(expectedRevision, previous = "") { + for (let attempt = 0; attempt < 90; attempt += 1) { + try { + const auth = JSON.parse(readFileSync(`${root}/config/internal-auth.json`)); + const response = await fetch(`http://127.0.0.1:${auth.port}/api/health`, { + headers: { "x-cg-secret": auth.secret }, signal: AbortSignal.timeout(2000), + }); + assert.equal(response.status, 200); + const value = await response.json(); + assert.equal(value.ok, true); + assert.equal(value.revision, expectedRevision); + assert.ok(value.instanceId && value.instanceId !== previous); + return value; + } catch { await new Promise((resolve) => setTimeout(resolve, 1000)); } + } + throw new Error(`Expected a new healthy instance on ${expectedRevision}`); +} +function restart() { + const pid = Number(run("systemctl", ["show", "--property", "MainPID", "--value", "channelgate.service"])); + assert.ok(Number.isInteger(pid) && pid > 1); + process.kill(pid, "SIGUSR2"); +} + +restart(); +let before = await health(baseline); +for (const failure of ["test", "readiness"]) { + if (failure === "test") { + const candidateManifest = { ...manifest, scripts: { ...manifest.scripts, test: 'node -e "process.exit(42)"' } }; + writeFileSync(packageFile, `${JSON.stringify(candidateManifest, null, 2)}\n`); + git("add", "package.json"); + } else { + const source = readFileSync(appFile, "utf8"); + assert.equal(source.split("revision: runningRevision,").length, 2); + writeFileSync(appFile, source.replace("revision: runningRevision,", 'revision: "lifecycle-intentionally-unready",')); + git("add", "src/web/app.js"); + } + git("commit", "-m", `fixture: intentional ${failure} failure`); + const candidate = git("rev-parse", "HEAD"); + // Only a new local fixture bare repository is written; no public remote is ever pushed. + git("push", "--force", "origin", "HEAD:lifecycle-update-fixture"); + git("reset", "--hard", baseline); + const result = spawnSync(process.execPath, ["scripts/update-runner.mjs"], { + cwd: repo, env: process.env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], + timeout: 12 * 60_000, maxBuffer: 4_000_000, + }); + const state = JSON.parse(readFileSync(`${root}/update-state.json`)); + // Only the fixed fixture/status fields are evidence; full authenticated health and runtime + // snapshots remain private inside the disposable VM. + console.log(JSON.stringify({ failure, updaterExit: result.status, result: state.result, + phase: state.phase, oldRevision: state.oldRevision, targetRevision: state.targetRevision, + runningRevision: state.runningRevision, candidateError: state.candidateError, rollbackError: state.rollbackError })); + assert.equal(result.error, undefined); + assert.equal(result.status, 2); + assert.equal(state.result, "rolled_back"); + assert.equal(state.oldRevision, baseline); + assert.equal(state.targetRevision, candidate); + assert.equal(state.runningRevision, baseline); + assert.equal(git("rev-parse", "HEAD"), baseline); + if (failure === "test") assert.match(state.candidateError, /npm test.*42/); + else assert.ok(state.candidateError.includes( + `gateway is running revision lifecycle-intentionally-unready instead of ${candidate}`), + "readiness failure must prove the candidate actually answered with the intended wrong revision"); + assert.equal(existsSync(`${root}/update-backups/${state.id}/gateway.db`), true); + assert.equal(JSON.parse(readFileSync(`${root}/update-backups/${state.id}/config/lifecycle-proof.json`)).value, "before-backup"); + const snapshot = new DatabaseSync(`${root}/update-backups/${state.id}/gateway.db`, { readOnly: true }); + assert.equal(snapshot.prepare("SELECT value FROM lifecycle_proof").get().value, "before-backup"); + assert.equal(snapshot.prepare("PRAGMA integrity_check").get().integrity_check, "ok"); + snapshot.close(); + assert.equal(existsSync(`${root}/update.lock`), false); + const db = new DatabaseSync(`${root}/gateway.db`, { readOnly: true }); + assert.equal(db.prepare("SELECT value FROM lifecycle_proof").get().value, "before-backup"); + db.close(); + before = await health(baseline, before.instanceId); + console.log(`PASS real ${failure} failure rolls back Git and restarts systemd; engine smoke is a controlled fixture`); +} +git("checkout", "--detach", original); +restart(); +await health(original, before.instanceId); +console.log("PASS original candidate restored after isolated updater fixture checks"); diff --git a/scripts/install-systemd.sh b/scripts/install-systemd.sh index 419384d..b1e9b41 100755 --- a/scripts/install-systemd.sh +++ b/scripts/install-systemd.sh @@ -117,11 +117,16 @@ fi # Probe and build in the final daemon identity's own rootless store. The installer's store and # image UID cannot be reused by a different service account. -run_as_service() { - runuser -u "$SERVICE_USER" -- env HOME="$SERVICE_HOME" CHANNELGATE_DIR="$SERVICE_HOME" \ +run_as_service() ( + # Match the unit's explicit environment. Inherited operator XDG/container-storage settings + # can point Podman at another user's private store even after HOME changes. + # Podman also re-enters cwd after its user-namespace transition; the invoking shell may be + # in an operator-private directory even when the installer itself lives under /opt. + cd "$APP_DIR" + runuser -u "$SERVICE_USER" -- env -i HOME="$SERVICE_HOME" CHANNELGATE_DIR="$SERVICE_HOME" \ XDG_RUNTIME_DIR="$SERVICE_RUNTIME_DIR" DBUS_SESSION_BUS_ADDRESS="unix:path=$SERVICE_RUNTIME_DIR/bus" \ PATH="$SERVICE_PATH" "$@" -} +) run_as_service podman info --format '{{.Host.Security.Rootless}}' | grep -qx true || { echo "Rootless Podman is not usable as $SERVICE_USER"; exit 1; } diff --git a/test/operations-readiness.test.js b/test/operations-readiness.test.js index 9d311c0..85bf1fd 100644 --- a/test/operations-readiness.test.js +++ b/test/operations-readiness.test.js @@ -138,6 +138,10 @@ test("service packages pin dedicated identities and hardened runtime boundaries" assert.match(systemd, /Delegate=yes/); assert.match(systemd, /--add-subids-for-system/); assert.match(systemd, /run_as_service "\$NODE_BIN"/); + assert.match(systemd, /runuser -u "\$SERVICE_USER" -- env -i HOME=/, + "image provisioning must not inherit an operator's XDG/container storage configuration"); + assert.match(systemd, /run_as_service\(\) \([\s\S]*?cd "\$APP_DIR"[\s\S]*?runuser/, + "Podman must start from the service-owned checkout, not the caller's private cwd"); assert.match(systemd, /Requires=user@\$SERVICE_UID\.service/); assert.match(systemd, /After=network-online.target user@\$SERVICE_UID\.service/); assert.ok(systemd.indexOf('scripts/service-path-preflight.mjs') < systemd.indexOf('useradd --system'));