diff --git a/.github/scripts/ubuntu_archive_dns.sh b/.github/scripts/ubuntu_archive_dns.sh new file mode 100755 index 000000000..4e4ee28d3 --- /dev/null +++ b/.github/scripts/ubuntu_archive_dns.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Resolve Ubuntu's archive to Azure's mirror of it for everything on this runner: its builds, its containers and the +# runner itself. A dnsmasq on Docker's bridge answers archive.ubuntu.com and security.ubuntu.com with the mirror's +# addresses and forwards every other name to the runner's own resolver; the runner's resolv.conf names it, and Docker +# copies that into each container and BuildKit into each RUN, since it isn't a loopback address. +set -euo pipefail +MIRROR=azure.archive.ubuntu.com +HOSTS=(archive.ubuntu.com security.ubuntu.com) +bridge=$(ip -4 addr show docker0 | awk '/inet /{print $2}' | cut -d/ -f1) +upstream=$(awk '/^nameserver/{print $2; exit}' /run/systemd/resolve/resolv.conf) +mirror=$(getent ahostsv4 "$MIRROR" | awk '{print $1; exit}') +[ -n "$bridge" ] && [ -n "$upstream" ] && [ -n "$mirror" ] || { echo "bridge=$bridge upstream=$upstream mirror=$mirror" >&2; exit 1; } +command -v dnsmasq >/dev/null || sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq dnsmasq-base >/dev/null +args=(--conf-file=/dev/null --pid-file=/run/agentenv-archive-dns.pid --listen-address="$bridge" --bind-dynamic + --no-resolv --server="$upstream" --filter-AAAA) +for host in "${HOSTS[@]}"; do args+=(--address="/$host/$mirror"); done +sudo dnsmasq "${args[@]}" +getent hosts "$(hostname)" >/dev/null || echo "127.0.1.1 $(hostname)" | sudo tee -a /etc/hosts >/dev/null +sudo rm -f /etc/resolv.conf +printf 'nameserver %s\noptions edns0\n' "$bridge" | sudo tee /etc/resolv.conf >/dev/null +for host in "${HOSTS[@]}"; do + got=$(getent ahostsv4 "$host" | awk '{print $1; exit}') + [ "$got" = "$mirror" ] || { echo "$host resolves to $got, not $MIRROR's $mirror" >&2; exit 1; } +done +getent ahostsv4 pypi.org >/dev/null || { echo "other names don't resolve" >&2; exit 1; } +echo "Ubuntu's archive resolves to $MIRROR ($mirror) through $bridge; other names through $upstream" diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index ee15008e6..3b0f3814c 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -143,6 +143,11 @@ jobs: python -m pip install --quiet -r .github/ci-requirements.txt uv sync --locked --extra dev --python "$(command -v python)" echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + # Ubuntu's archive stopped answering for hours at a time on 11-19, failing every build and compose that runs apt, + # on main too, while the Azure mirror of it the runners use stayed up. Builds, containers and the runner resolve + # the archive to that mirror. + - name: Resolve Ubuntu's archive to Azure's mirror + run: bash .github/scripts/ubuntu_archive_dns.sh # loadscope keeps a module's or class's tests on one worker, so its fixtures run once, as make int-test-fast does. - name: Integration tier on the local defaults run: | @@ -190,6 +195,11 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst + # Ubuntu's archive stopped answering for hours at a time on 11-19, failing every build and compose that runs apt, + # on main too, while the Azure mirror of it the runners use stayed up. Builds, containers and the runner resolve + # the archive to that mirror. + - name: Resolve Ubuntu's archive to Azure's mirror + run: bash .github/scripts/ubuntu_archive_dns.sh # The computer tests share the package locks a compose resolves, and the files they name, in the packages folder # (tst/util/package_cache.py). An image's key includes its lock, which the composer otherwise resolves from # Ubuntu's archive, PyPI and download URLs, so the folder is restored from what main saved latest this week. @@ -280,6 +290,11 @@ jobs: echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst + # Ubuntu's archive stopped answering for hours at a time on 11-19, failing every build and compose that runs apt, + # on main too, while the Azure mirror of it the runners use stayed up. Builds, containers and the runner resolve + # the archive to that mirror. + - name: Resolve Ubuntu's archive to Azure's mirror + run: bash .github/scripts/ubuntu_archive_dns.sh - name: Start the local registry run: | docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2