From 09427ce8ce6e884d563f943ccebab0d407b2dc33 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 19 Nov 2026 04:55:06 -0800 Subject: [PATCH] ci: resolve Ubuntu's archive to Azure's mirror of it in the integration jobs Ubuntu's archive stopped answering for minutes to hours at a time on 11-19 (index files taking 20-40 s, then no answer at all), failing every image build and compose that runs apt in the integration tiers, on main too, while azure.archive.ubuntu.com, the mirror the runners' own apt uses, stayed up. A step now runs dnsmasq on Docker's bridge, answering archive.ubuntu.com and security.ubuntu.com with that mirror's address and forwarding every other name to the runner's resolver, and points the runner's resolv.conf at it; Docker copies that into each container, and BuildKit into each RUN, since it isn't a loopback address. The mirror answers those hosts' names. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/ubuntu_archive_dns.sh | 26 ++++++++++++++++++++++++++ .github/workflows/local-backends.yml | 15 +++++++++++++++ 2 files changed, 41 insertions(+) create mode 100755 .github/scripts/ubuntu_archive_dns.sh diff --git a/.github/scripts/ubuntu_archive_dns.sh b/.github/scripts/ubuntu_archive_dns.sh new file mode 100755 index 000000000..4e4ee28d3 --- /dev/null +++ b/.github/scripts/ubuntu_archive_dns.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Resolve Ubuntu's archive to Azure's mirror of it for everything on this runner: its builds, its containers and the +# runner itself. A dnsmasq on Docker's bridge answers archive.ubuntu.com and security.ubuntu.com with the mirror's +# addresses and forwards every other name to the runner's own resolver; the runner's resolv.conf names it, and Docker +# copies that into each container and BuildKit into each RUN, since it isn't a loopback address. +set -euo pipefail +MIRROR=azure.archive.ubuntu.com +HOSTS=(archive.ubuntu.com security.ubuntu.com) +bridge=$(ip -4 addr show docker0 | awk '/inet /{print $2}' | cut -d/ -f1) +upstream=$(awk '/^nameserver/{print $2; exit}' /run/systemd/resolve/resolv.conf) +mirror=$(getent ahostsv4 "$MIRROR" | awk '{print $1; exit}') +[ -n "$bridge" ] && [ -n "$upstream" ] && [ -n "$mirror" ] || { echo "bridge=$bridge upstream=$upstream mirror=$mirror" >&2; exit 1; } +command -v dnsmasq >/dev/null || sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq dnsmasq-base >/dev/null +args=(--conf-file=/dev/null --pid-file=/run/agentenv-archive-dns.pid --listen-address="$bridge" --bind-dynamic + --no-resolv --server="$upstream" --filter-AAAA) +for host in "${HOSTS[@]}"; do args+=(--address="/$host/$mirror"); done +sudo dnsmasq "${args[@]}" +getent hosts "$(hostname)" >/dev/null || echo "127.0.1.1 $(hostname)" | sudo tee -a /etc/hosts >/dev/null +sudo rm -f /etc/resolv.conf +printf 'nameserver %s\noptions edns0\n' "$bridge" | sudo tee /etc/resolv.conf >/dev/null +for host in "${HOSTS[@]}"; do + got=$(getent ahostsv4 "$host" | awk '{print $1; exit}') + [ "$got" = "$mirror" ] || { echo "$host resolves to $got, not $MIRROR's $mirror" >&2; exit 1; } +done +getent ahostsv4 pypi.org >/dev/null || { echo "other names don't resolve" >&2; exit 1; } +echo "Ubuntu's archive resolves to $MIRROR ($mirror) through $bridge; other names through $upstream" diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index ee15008e6..3b0f3814c 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -143,6 +143,11 @@ jobs: python -m pip install --quiet -r .github/ci-requirements.txt uv sync --locked --extra dev --python "$(command -v python)" echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + # Ubuntu's archive stopped answering for hours at a time on 11-19, failing every build and compose that runs apt, + # on main too, while the Azure mirror of it the runners use stayed up. Builds, containers and the runner resolve + # the archive to that mirror. + - name: Resolve Ubuntu's archive to Azure's mirror + run: bash .github/scripts/ubuntu_archive_dns.sh # loadscope keeps a module's or class's tests on one worker, so its fixtures run once, as make int-test-fast does. - name: Integration tier on the local defaults run: | @@ -190,6 +195,11 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst + # Ubuntu's archive stopped answering for hours at a time on 11-19, failing every build and compose that runs apt, + # on main too, while the Azure mirror of it the runners use stayed up. Builds, containers and the runner resolve + # the archive to that mirror. + - name: Resolve Ubuntu's archive to Azure's mirror + run: bash .github/scripts/ubuntu_archive_dns.sh # The computer tests share the package locks a compose resolves, and the files they name, in the packages folder # (tst/util/package_cache.py). An image's key includes its lock, which the composer otherwise resolves from # Ubuntu's archive, PyPI and download URLs, so the folder is restored from what main saved latest this week. @@ -280,6 +290,11 @@ jobs: echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst + # Ubuntu's archive stopped answering for hours at a time on 11-19, failing every build and compose that runs apt, + # on main too, while the Azure mirror of it the runners use stayed up. Builds, containers and the runner resolve + # the archive to that mirror. + - name: Resolve Ubuntu's archive to Azure's mirror + run: bash .github/scripts/ubuntu_archive_dns.sh - name: Start the local registry run: | docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2