diff --git a/.github/actions/truenas-appliance-runner/action.yml b/.github/actions/truenas-appliance-runner/action.yml index e871e46..6221b6b 100644 --- a/.github/actions/truenas-appliance-runner/action.yml +++ b/.github/actions/truenas-appliance-runner/action.yml @@ -30,6 +30,13 @@ description: >- # action sets them; truenas-appliance (appliance.sh) is on PATH for # snapshot, revert, probe, call and console. # +# `network: bridge` puts the appliance on a bridge with the runner instead +# of behind qemu's NAT: TN_HOST is its own address, 10.232.0.10, on port +# 443, which the runner, its containers and a browser reach directly, and +# the appliance reaches the runner at TN_RUNNER_IP, 10.232.0.1. For what +# needs the appliance's real address and ports, such as a TrueNAS Connect +# stack on the runner. The default, `user`, is unchanged. +# # `ha: true` boots an HA pair instead (ha.sh): two appliances, from a pool # that attaches the appliance disk twice, # @@ -76,6 +83,15 @@ inputs: apply interface changes on without cutting itself off. Their MACs, 52:54:00:e2:e0:01 and up, are in TN_SPARE_NICS. Not for an HA pair. default: '0' + network: + description: >- + `user`: behind qemu's NAT, its ports forwarded to the runner's + loopback (TN_HOST 127.0.0.1:8443), and reaching the runner's loopback + at 10.0.2.2. `bridge`: on a bridge with the runner, at 10.232.0.10 on + its real ports (TN_HOST 10.232.0.10:443), reached directly from the + runner, its containers and a browser, and reaching what the runner + serves on 10.232.0.1. Needs passwordless sudo. Not for an HA pair. + default: user ssh: description: '`true` enables ssh for the admin, on TN_SSH.' default: 'false' @@ -102,6 +118,18 @@ outputs: ssh: description: host:port of its ssh, when asked for. value: ${{ steps.start.outputs.ssh }} + network: + description: '`user` or `bridge`, as asked; empty for an HA pair.' + value: ${{ steps.start.outputs.network }} + runner-ip: + description: >- + Where the appliance reaches the runner: 10.0.2.2 with `user`, which + qemu maps to the runner's loopback, or 10.232.0.1 with `bridge`. + Empty for an HA pair. + value: ${{ steps.start.outputs.runner-ip }} + appliance-ip: + description: The appliance's own address with `bridge`, 10.232.0.10; empty otherwise. + value: ${{ steps.start.outputs.appliance-ip }} boot-seconds: description: From qemu starting to the API taking a login. value: ${{ steps.start.outputs.boot-seconds }} @@ -122,6 +150,7 @@ runs: APPLIANCE_DATA_DISK_GB: ${{ inputs.data-disk-gb }} APPLIANCE_EXTRA_DISK_GB: ${{ inputs.extra-disk-gb }} APPLIANCE_SPARE_NICS: ${{ inputs.spare-nics }} + APPLIANCE_NETWORK: ${{ inputs.network }} APPLIANCE_SSH: ${{ inputs.ssh }} APPLIANCE_DISK_INPUT: ${{ inputs.disk }} EXPORT_ENV: ${{ inputs.export-env }} @@ -140,6 +169,10 @@ runs: echo "::error::spare-nics is for one appliance; ha.sh wires an HA pair's NICs itself" exit 1 fi + if [ "$HA" = true ] && [ "${APPLIANCE_NETWORK:-user}" != user ]; then + echo "::error::network is for one appliance; ha.sh puts an HA pair on bridges of its own" + exit 1 + fi export APPLIANCE_DIR="$RUNNER_TEMP/truenas-appliance" # The password is made here, so it is masked before anything can # print it. @@ -169,6 +202,9 @@ runs: echo "username=$(get TN_USERNAME)" echo "password=$(get TN_PASSWORD)" echo "ssh=$(get TN_SSH)" + echo "network=$(get TN_NETWORK)" + echo "runner-ip=$(get TN_RUNNER_IP)" + echo "appliance-ip=$(get TN_APPLIANCE_IP)" echo "boot-seconds=$(get APPLIANCE_BOOT_SECONDS)" echo "snapshot-seconds=$(get APPLIANCE_SNAPSHOT_SECONDS)" } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/truenas-appliance-runner/appliance.sh b/.github/actions/truenas-appliance-runner/appliance.sh index d00f244..4cc7d74 100755 --- a/.github/actions/truenas-appliance-runner/appliance.sh +++ b/.github/actions/truenas-appliance-runner/appliance.sh @@ -25,7 +25,7 @@ # result as JSON on stdout # appliance.sh snapshots the names there are # appliance.sh console [lines] the end of its serial console -# appliance.sh stop +# appliance.sh stop quit it, and take its bridge network down # # Settings, environment variables, all optional: # @@ -42,8 +42,16 @@ # alone on a qemu hub of its own, so no DHCP, no # route and no peer: for a suite to reconfigure # without cutting itself off +# APPLIANCE_NETWORK user: qemu's user-mode network, the guest behind +# its NAT and its ports forwarded to this machine's +# loopback, on the ports below. bridge: the guest +# on a bridge with this machine (bridge_up), on its +# own address and its real ports, reachable from +# here, from containers and from a browser; needs +# passwordless sudo # APPLIANCE_HTTP_PORT 8080 APPLIANCE_HTTPS_PORT 8443 # APPLIANCE_SSH_PORT 2222 APPLIANCE_SSH false +# (the ports forwarded with user; bridge has none) # APPLIANCE_BASELINE fresh-install: the snapshot start takes; empty # takes none # APPLIANCE_PASSWORD the claim's admin password (default: generated) @@ -55,8 +63,10 @@ # TN_USERNAME, TN_PASSWORD, TN_DOMAIN, TN_BASELINE (the suite's contract, # as the lab's claim action gives it, less its TN_GUEST_ISO), TN_SSH when # APPLIANCE_SSH is true, TN_SPARE_NICS (the spare NICs' MAC addresses, -# space-separated) when there are any, and what it measured: -# APPLIANCE_BOOT_SECONDS, APPLIANCE_SNAPSHOT_SECONDS. +# space-separated) when there are any, TN_NETWORK (user or bridge), +# TN_RUNNER_IP (where the guest reaches this machine), TN_APPLIANCE_IP +# with bridge, and what it measured: APPLIANCE_BOOT_SECONDS, +# APPLIANCE_SNAPSHOT_SECONDS. set -euo pipefail # shellcheck source=lib.sh @@ -69,11 +79,47 @@ QMP="$DIR/qmp.sock" QGA="$DIR/qga.sock" STATE="$DIR/state.env" +# The bridge network (APPLIANCE_NETWORK=bridge): this machine at .1 and the +# appliance at .10, by its MAC. Not ha.sh's 10.231.0.0/24, Docker's +# 172.17.0.0/16 or compose's networks. The MAC is fixed so that the address +# is the same after a revert; the spare NICs' are e0:01 and up. +BR=tnapp-br +TAP=tnapp-tap +BR_NET=10.232.0 +BR_RUNNER_IP=$BR_NET.1 +BR_APPLIANCE_IP=$BR_NET.10 +BR_MAC=52:54:00:e2:e0:00 +# What bridge_up changed, written before it changes anything: bridge_down +# reads it, even after a start that failed halfway. +NETSTATE="$DIR/network.env" +# The rules bridge_up adds, as "table chain rule...". Docker loads +# br_netfilter and drops FORWARD by default, which would drop the +# appliance's traffic to containers and beyond this machine, and theirs to +# it; the NAT takes it beyond this machine, as qemu's user-mode NAT does. +BR_RULES=( + "filter FORWARD -i $BR -j ACCEPT" + "filter FORWARD -o $BR -j ACCEPT" + "nat POSTROUTING -s $BR_NET.0/24 ! -o $BR -j MASQUERADE" +) + hmp() { "${TNCTL[@]}" hmp "$QMP" "$*"; } # resume: a failed loadvm leaves the VM paused, and later loadvms leave it # so, however they go; cont is a no-op on one that runs. resume() { "${TNCTL[@]}" qmp "$QMP" cont >/dev/null; } -url() { echo "ws://127.0.0.1:${APPLIANCE_HTTP_PORT:-8080}"; } +# addr : host:port where this machine reaches that port of the +# appliance: its own address on the bridge, or qemu's forward of it. +addr() { + if [ "${APPLIANCE_NETWORK:-user}" = bridge ]; then + echo "$BR_APPLIANCE_IP:$1" + return + fi + case "$1" in + 80) echo "127.0.0.1:${APPLIANCE_HTTP_PORT:-8080}" ;; + 443) echo "127.0.0.1:${APPLIANCE_HTTPS_PORT:-8443}" ;; + 22) echo "127.0.0.1:${APPLIANCE_SSH_PORT:-2222}" ;; + esac +} +url() { echo "ws://$(addr 80)"; } running() { [ -f "$DIR/qemu.pid" ] && kill -0 "$(cat "$DIR/qemu.pid")" 2>/dev/null; } # shellcheck source=/dev/null state() { [ -r "$STATE" ] || die "no appliance started here ($STATE)"; . "$STATE"; } @@ -104,6 +150,70 @@ print(found[0]) ' } +# bridge_up: the bridge network. A bridge with this machine on it, a tap +# for qemu (this user's, so qemu runs without sudo), BR_RULES, and dnsmasq +# giving the appliance's MAC its address and answering its DNS. +bridge_up() { + say "network: bridge $BR, this machine at $BR_RUNNER_IP, the appliance at $BR_APPLIANCE_IP" + local forward rule r + forward="$(cat /proc/sys/net/ipv4/ip_forward)" + echo "BR_FORWARD_WAS=$forward" > "$NETSTATE" + sudo -n ip link add "$BR" type bridge + sudo -n ip addr add "$BR_RUNNER_IP/24" dev "$BR" + sudo -n ip link set "$BR" up + sudo -n ip tuntap add dev "$TAP" mode tap user "$(id -u)" + sudo -n ip link set "$TAP" master "$BR" up + for rule in "${BR_RULES[@]}"; do + read -r -a r <<< "$rule" + sudo -n iptables -t "${r[0]}" -I "${r[1]}" "${r[@]:2}" + done + # Docker turns it on; a runner without Docker may not have. + if [ "$forward" != 1 ]; then + sudo -n sysctl -q -w net.ipv4.ip_forward=1 + fi + # DNS as well as DHCP, unlike ha.sh's: the appliance needs a resolver + # for NTP and for what middleware looks up as it starts, and this + # machine's may be a stub on its loopback (systemd-resolved's + # 127.0.0.53) that the appliance cannot reach. dnsmasq forwards to + # whatever /etc/resolv.conf names, and its offer names it as the router + # and the DNS server; --no-hosts keeps this machine's /etc/hosts out of + # its answers. Static: it answers this MAC alone. The lease never ends, + # so a revert, which restores the guest's lease with its memory and + # moves its clock on, never finds it expired. + sudo -n dnsmasq --interface="$BR" --bind-interfaces --except-interface=lo --no-hosts \ + --dhcp-range="$BR_APPLIANCE_IP,static" --dhcp-host="$BR_MAC,$BR_APPLIANCE_IP,infinite" \ + --dhcp-leasefile="$DIR/dnsmasq.leases" --pid-file="$DIR/dnsmasq.pid" \ + --log-dhcp --log-facility="$DIR/dnsmasq.log" +} + +# bridge_down: undo bridge_up, each part only if it is there, so that a +# self-hosted runner's next start finds nothing of this one. Nothing to do +# for a user-mode appliance. +bridge_down() { + if [ ! -f "$NETSTATE" ] && ! ip link show "$BR" >/dev/null 2>&1; then + return 0 + fi + say "network: taking $BR down" + local BR_FORWARD_WAS=1 rule r + # shellcheck source=/dev/null + if [ -r "$NETSTATE" ]; then . "$NETSTATE"; fi + if [ -s "$DIR/dnsmasq.pid" ]; then + sudo -n kill "$(cat "$DIR/dnsmasq.pid")" 2>/dev/null || true + fi + for rule in "${BR_RULES[@]}"; do + read -r -a r <<< "$rule" + while sudo -n iptables -t "${r[0]}" -C "${r[1]}" "${r[@]:2}" 2>/dev/null; do + sudo -n iptables -t "${r[0]}" -D "${r[1]}" "${r[@]:2}" + done + done + if ip link show "$TAP" >/dev/null 2>&1; then sudo -n ip link del "$TAP"; fi + if ip link show "$BR" >/dev/null 2>&1; then sudo -n ip link del "$BR"; fi + if [ "$BR_FORWARD_WAS" != 1 ]; then + sudo -n sysctl -q -w net.ipv4.ip_forward="$BR_FORWARD_WAS" + fi + rm -f "$NETSTATE" +} + start() { local disk="${APPLIANCE_DISK:-}" local memory="${APPLIANCE_MEMORY_MB:-6144}" vcpus="${APPLIANCE_VCPUS:-4}" @@ -117,6 +227,18 @@ start() { local name="appliance-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-job}" running && die "an appliance is running already ($DIR)" + APPLIANCE_NETWORK="${APPLIANCE_NETWORK:-user}" + case "$APPLIANCE_NETWORK" in + user) ;; + bridge) + if ip link show "$BR" >/dev/null 2>&1; then + die "$BR exists already: an earlier appliance's network is still up, and truenas-appliance stop takes it down" + fi + sudo -n true 2>/dev/null || die "APPLIANCE_NETWORK=bridge needs passwordless sudo, for the bridge, the tap, iptables and dnsmasq" + needs dnsmasq dnsmasq-base iptables iptables + ;; + *) die "APPLIANCE_NETWORK: \"$APPLIANCE_NETWORK\" is not user or bridge" ;; + esac local gb for gb in "${extragb[@]}"; do [[ "$gb" =~ ^[1-9][0-9]*$ ]] || die "APPLIANCE_EXTRA_DISK_GB: \"$gb\" is not a whole number of GB" @@ -185,6 +307,15 @@ start() { elif [ "$nspare" -gt 1 ]; then nicssaid=", $nspare spare NICs" fi + # -nic either way, so the NIC is made the same way, keeps its PCI slot, + # and with it its name in the guest. + local nic + if [ "$APPLIANCE_NETWORK" = bridge ]; then + bridge_up + nic="tap,ifname=$TAP,script=no,downscript=no,model=virtio-net-pci,mac=$BR_MAC" + else + nic="user,model=virtio-net-pci$(hostfwd 80 "$http")$(hostfwd 443 "$https")$(hostfwd 22 "$sshp")" + fi local booted t0 say "booting $disk as $name: ${memory} MB, $vcpus vCPUs, $ndata data disks of ${datagb}G$extrasaid$nicssaid" @@ -196,7 +327,7 @@ start() { -drive "file=$DIR/os.qcow2,if=none,id=os,format=qcow2,discard=unmap" \ -device "virtio-blk-pci,drive=os,serial=os,bootindex=0" \ "${drives[@]}" \ - -nic "user,model=virtio-net-pci$(hostfwd 80 "$http")$(hostfwd 443 "$https")$(hostfwd 22 "$sshp")" \ + -nic "$nic" \ "${nics[@]}" \ -device virtio-rng-pci \ -device virtio-serial-pci,id=vser0 \ @@ -215,16 +346,21 @@ start() { --since "$t0" --timeout "${APPLIANCE_BOOT_TIMEOUT:-900}"); then say "--- its console, last 60 lines" console 60 >&2 || true + if [ "$APPLIANCE_NETWORK" = bridge ]; then + say "--- dnsmasq" + cat "$DIR/dnsmasq.log" >&2 2>/dev/null || true + fi die "the appliance's API never came up" fi say "API up ${booted}s after qemu started" "${TNCTL[@]}" rotate "$(url)" "$APPLIANCE_USER" "$APPLIANCE_TEMPLATE_PASSWORD" "$password" if [ "${APPLIANCE_SSH:-false}" = true ]; then "${TNCTL[@]}" enable-ssh "$(url)" "$APPLIANCE_USER" "$password" - say "ssh on 127.0.0.1:$sshp" + say "ssh on $(addr 22)" fi { echo "APPLIANCE_PASSWORD_NOW=$(printf %q "$password")" + echo "APPLIANCE_NETWORK=$APPLIANCE_NETWORK" echo "APPLIANCE_HTTP_PORT=$http" echo "APPLIANCE_SSH_PORT=$sshp" } > "$STATE" @@ -238,21 +374,31 @@ start() { say "snapshot $baseline taken in ${snapped}s" fi + # TN_RUNNER_IP: where the guest reaches this machine. Under qemu's NAT + # that is 10.0.2.2, which qemu maps to this machine's loopback; on the + # bridge, what listens on the bridge's address or on every address. + local runner_ip=10.0.2.2 + [ "$APPLIANCE_NETWORK" = bridge ] && runner_ip=$BR_RUNNER_IP cat <&2 - local port="${APPLIANCE_SSH_PORT:-2222}" + local at + at="$(addr 22)" local -a ssh_ - ssh_=(ssh -i "$key" -p "$port" -o BatchMode=yes -o ConnectTimeout=5 + ssh_=(ssh -i "$key" -p "${at##*:}" -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR - "$APPLIANCE_USER@127.0.0.1") + "$APPLIANCE_USER@${at%:*}") local i for i in $(seq 1 30); do "${ssh_[@]}" true 2>/dev/null && break - [ "$i" = 30 ] && die "no ssh into the appliance on 127.0.0.1:$port" + [ "$i" = 30 ] && die "no ssh into the appliance on $at" sleep 2 done "${ssh_[@]}" 'bash -s' <<'REPORT' @@ -394,6 +541,21 @@ print(f"{ms / 1000 - time.time():.1f}")')" echo "APPLIANCE_REVERT_SECONDS=$took" } +# stop: quit qemu, then take down the bridge network, if there is one: the +# tap is qemu's until it exits. +stop() { + if running; then + local pid i + pid="$(cat "$DIR/qemu.pid")" + "${TNCTL[@]}" qmp "$QMP" quit >/dev/null || true + for i in $(seq 1 50); do + kill -0 "$pid" 2>/dev/null || break + sleep 0.2 + done + fi + bridge_down +} + case "${1:-}" in start) start ;; call) shift; [ $# -gt 0 ] || die "call [params...]"; state @@ -404,6 +566,6 @@ revert) shift; revert "$@" ;; snapshots) running || die "no appliance running"; hmp info snapshots ;; console) console "${2:-100}" ;; boot-report) boot_report ;; -stop) if running; then "${TNCTL[@]}" qmp "$QMP" quit >/dev/null || true; fi ;; +stop) stop ;; *) die "usage: appliance.sh start | snapshot | revert | probe [baseline] | call [params...] | boot-report | snapshots | console [lines] | stop" ;; esac diff --git a/README.md b/README.md index d225467..ad03081 100644 --- a/README.md +++ b/README.md @@ -564,16 +564,51 @@ jobs: | `data-disk-count`, `data-disk-gb` | `10`, `10` | Sparse qcow2 files | | `extra-disk-gb` | none | More sparse disks after the data disks, one per size in GB: `'32'`, `'12 12 15'`. Not for an HA pair | | `spare-nics` | `'0'` | NICs after the one the API is reached on, each alone on a qemu hub: no DHCP, no route. For applying interface changes without losing the appliance. MACs `52:54:00:e2:e0:01` and up, in `TN_SPARE_NICS`. Not for an HA pair | +| `network` | `user` | `user`: behind qemu's NAT, its ports forwarded to the runner's loopback. `bridge`: on a bridge with the runner, on its own address and real ports; see below. Not for an HA pair | | `ssh` | `'false'` | `true` enables ssh for the admin, on `TN_SSH` | | `disk` | the attached one | Found by its serial, `virtio-truenas_appliance`, where the box keeps it; TrueNAS replaces it, so otherwise the one disk besides the root disk that holds a ZFS `boot-pool` | | `export-env` | `'true'` | Write `TN_*` to the job environment | Outputs: `host`, `host-http`, `username`, `password` (masked), `ssh`, -`boot-seconds`, `snapshot-seconds`. `truenas-appliance` is put on `PATH`: -`snapshot `, `revert `, `call [params...]`, -`snapshots`, `console [lines]`, `probe`, and `boot-report`: where the boot's -time went, from systemd and middleware's own log, over ssh (it leaves ssh on -with a key and sudo for the admin, so revert afterwards if that matters). +`network`, `runner-ip`, `appliance-ip`, `boot-seconds`, `snapshot-seconds`. +`truenas-appliance` is put on `PATH`: `snapshot `, `revert `, +`call [params...]`, `snapshots`, `console [lines]`, `probe`, +`boot-report`: where the boot's time went, from systemd and middleware's own +log, over ssh (it leaves ssh on with a key and sudo for the admin, so revert +afterwards if that matters), and `stop`. + +| Variable | `network: user` | `network: bridge` | +|---|---|---| +| `TN_HOST` | `127.0.0.1:8443` | `10.232.0.10:443` | +| `TN_HOST_HTTP` | `127.0.0.1:8080` | `10.232.0.10:80` | +| `TN_SSH` (with `ssh: true`) | `127.0.0.1:2222` | `10.232.0.10:22` | +| `TN_NETWORK` | `user` | `bridge` | +| `TN_RUNNER_IP`: where the appliance reaches the runner | `10.0.2.2` | `10.232.0.1` | +| `TN_APPLIANCE_IP` | not set | `10.232.0.10` | + +**`network: bridge`** is for what needs the appliance's own address and real +ports, such as a TrueNAS Connect stack on the runner: registration publishes +the NAS's own IPs, and a client then connects to them on 443. The runner is +`10.232.0.1` on a bridge, `tnapp-br`; qemu's NIC is a tap on it with the +fixed MAC `52:54:00:e2:e0:00`, and dnsmasq gives that MAC `10.232.0.10` for +good and forwards the appliance's DNS to the runner's resolvers. iptables +accepts the bridge's traffic through Docker's `FORWARD` policy and NATs the +appliance out, as qemu's NAT does under `user`. The appliance is reached on +`10.232.0.10` from the runner, from containers on Docker's networks and from +a browser on the runner, and keeps its address across `revert`. + +- **Nothing is forwarded to `127.0.0.1`** with `bridge`. A consumer that + hard-codes it, or reaches the appliance as `host.docker.internal` + (webui's nginx container), keeps `user`. +- **What the appliance reaches on the runner differs.** Under `user`, + `10.0.2.2` is the runner's loopback, so a port published on `127.0.0.1` + is reachable. On the bridge, the appliance reaches what listens on + `10.232.0.1` or on every address; publish a container's port there. +- **It needs passwordless sudo**, for the bridge, the tap, iptables and + dnsmasq (GARM's runners have it); qemu itself still runs as the job's user. +- **`truenas-appliance stop`** quits qemu and removes the bridge, the tap, + dnsmasq and the iptables rules. `start` refuses while `tnapp-br` exists. + An ephemeral runner needs neither; a self-hosted one re-running jobs does. `revert` restores the appliance's memory, and with it the clock it had when the snapshot was taken. It then sets the clock to the runner's through the