From 63659034fc1a192b0dc609e3d2da5341c95c4419 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Wed, 5 Aug 2026 10:42:46 -0400 Subject: [PATCH] Drop pypi server, its diagnostic instrumentation, and the connectivity probe Package hosting has moved to halide/pypi (GitHub Releases + Pages) since this VM's network path turned out to be subject to intermittent MIT border-security IP quarantines outside anyone's control. Removes the pypiserver service, its Caddy vhost, the netmon sidecar and caddy access logging that were added specifically to diagnose that issue, and the standalone probe-connection.yml diagnostic workflow. --- .github/workflows/probe-connection.yml | 10 --- README.md | 34 +++++----- caddy/Caddyfile | 43 ------------ docker-compose.yml | 34 ---------- monitor/sample.sh | 91 -------------------------- 5 files changed, 16 insertions(+), 196 deletions(-) delete mode 100644 .github/workflows/probe-connection.yml delete mode 100755 monitor/sample.sh diff --git a/.github/workflows/probe-connection.yml b/.github/workflows/probe-connection.yml deleted file mode 100644 index bfaab2f..0000000 --- a/.github/workflows/probe-connection.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: probe -on: workflow_dispatch -jobs: - reach: - runs-on: ubuntu-slim - steps: - - run: | - echo "egress IP:"; curl -sS https://api.ipify.org; echo - echo "--- connect test ---" - curl -4 -v --connect-timeout 20 https://pypi.halide-lang.org/ 2>&1 | tail -25 diff --git a/README.md b/README.md index adbaf26..4373297 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,24 @@ $ uv sync --all-packages # Master configuration -The master is deployed via [Docker Compose][dc], which manages five services: -a PostgreSQL database, the Buildbot master, a [pypiserver] instance for hosting -`halide-llvm` wheels, a cleanup sidecar that evicts old development wheels, and -a [Caddy] reverse proxy with automatic HTTPS. +The master is deployed via [Docker Compose][dc], which manages three services: +a PostgreSQL database, the Buildbot master, and a [Caddy] reverse proxy with +automatic HTTPS. + +`halide-llvm` (and other) wheels used to be hosted here via a bundled +[pypiserver] instance at `pypi.halide-lang.org`, fronted by this same Caddy. +That's gone: the MIT-hosted VM this appliance runs on turned out to be subject +to intermittent border-security IP quarantines outside anyone's control +(shared cloud egress IPs occasionally get flagged for unrelated scanning +activity from other tenants, with no exception mechanism available). Package +hosting has moved to [halide/pypi] (GitHub Releases + Pages), which routes CI +traffic off that network path entirely. `pypi.halide-lang.org` now points +there via DNS; this repo no longer serves it. ## Secrets -Six secrets control authentication with external users and servers. These will -need to be determined before starting up a new master. +Five secrets control authentication with external users and servers. These +will need to be determined before starting up a new master. 1. Obtain a [GitHub personal access token](https://github.com/settings/tokens) with at least the `repo` scope enabled (other scopes that are not currently @@ -32,8 +41,6 @@ need to be determined before starting up a new master. This is only needed when using PostgreSQL (i.e., when `HALIDE_BB_MASTER_DB_URL` contains `{DB_PASSWORD}`). The default SQLite backend does not require it. -6. Generate a password for uploading packages to the PyPI server. Call this - `PYPI_PASSWORD`. A convenient command for generating a secure secret is `openssl rand -hex 20`. @@ -45,7 +52,6 @@ $ echo "$WORKER_SECRET" > secrets/halide_bb_pass.txt $ echo "$WEBHOOK_SECRET" > secrets/webhook_token.txt $ echo "$WWW_PASSWORD" > secrets/buildbot_www_pass.txt $ echo "$DB_PASSWORD" > secrets/db_password.txt -$ docker run --rm httpd:2-alpine htpasswd -Bbn upload "$PYPI_PASSWORD" > secrets/pypi_htpasswd.txt ``` ## GitHub configuration @@ -64,15 +70,6 @@ the correct ones: ## Starting the master -Optionally choose a directory to hold PyPI packages (the default is -`./data/packages`): - -```console -$ export HALIDE_BB_PYPI_PACKAGES_DIR=$HOME/wheels -``` - -Then start all services: - ```console $ docker compose up -d --build ``` @@ -145,5 +142,6 @@ the worker to start automatically are provided under `worker/`: [Caddy]: https://caddyserver.com [dc]: https://docs.docker.com/compose/ +[halide/pypi]: https://github.com/halide/pypi [pypiserver]: https://github.com/pypiserver/pypiserver [uv]: https://docs.astral.sh/uv diff --git a/caddy/Caddyfile b/caddy/Caddyfile index a4825ad..ea5e91f 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -7,49 +7,6 @@ metrics } -pypi.halide-lang.org { - # Access log for every request that reaches Caddy. This is the key signal - # for the intermittent wheel-download failures: a truncated download shows - # up as status 200 with a `size` smaller than the wheel's content-length, - # whereas a client-side connect timeout never appears here at all (which - # would point at the network path rather than the server). Written to a - # dedicated volume so it is not interleaved with Caddy's TLS/ACME logs and - # is not rolled off by Docker's log driver. - log { - output file /var/log/caddy/access.log { - roll_size 100MiB - roll_keep 20 - roll_keep_for 2160h - } - format json - } - - # Serve the (large, static) wheel files directly from disk so that - # downloads don't tie up pypiserver's small Waitress thread pool. Only - # the generated indexes and uploads (POST) still hit the Python server. - # - # The bare /packages/ path is the index listing that pypiserver renders, - # so it must keep going to the backend; only requests for actual files - # (/packages/) are served statically. handle blocks are mutually - # exclusive and evaluated in order, so the exact-match listing wins first. - handle /packages/ { - reverse_proxy pypi:8080 - } - - handle_path /packages/* { - root * /wheels - file_server - } - - reverse_proxy pypi:8080 - - header { - Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" - X-Frame-Options DENY - X-Content-Type-Options nosniff - } -} - buildbot.halide-lang.org { redir / /master/ temporary diff --git a/docker-compose.yml b/docker-compose.yml index 15e36a7..9ec63f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,17 +39,6 @@ services: driver: local restart: unless-stopped - pypi: - image: pypiserver/pypiserver:v2.4.0 - volumes: - - ${HALIDE_BB_PYPI_PACKAGES_DIR:-./data/packages}:/data/packages - secrets: - - pypi_htpasswd.txt - command: run -P /run/secrets/pypi_htpasswd.txt -a update --backend cached-dir /data/packages - logging: - driver: local - restart: unless-stopped - caddy: image: caddy:2-alpine # --watch reloads the Caddyfile on change. It works only because the @@ -65,27 +54,8 @@ services: volumes: - ./caddy:/etc/caddy:ro - ./www:/srv:ro - - ${HALIDE_BB_PYPI_PACKAGES_DIR:-./data/packages}:/wheels:ro - caddy-data:/data - caddy-config:/config - - caddy-logs:/var/log/caddy - logging: - driver: local - restart: unless-stopped - - # Host-level TCP/network sampler for diagnosing the intermittent - # wheel-download connect timeouts. network_mode: host so the /proc/net - # counters it reads reflect the host stack that terminates inbound :443, - # not this container's own namespace. See monitor/sample.sh. - netmon: - image: alpine:3.20 - network_mode: host - command: sh /monitor/sample.sh - volumes: - - ./monitor:/monitor:ro - - netmon-logs:/var/log/netmon - environment: - SAMPLE_INTERVAL: "15" logging: driver: local restart: unless-stopped @@ -93,8 +63,6 @@ services: volumes: caddy-data: caddy-config: - caddy-logs: - netmon-logs: db-data: secrets: @@ -108,5 +76,3 @@ secrets: file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/webhook_token.txt buildbot_www_pass.txt: file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/buildbot_www_pass.txt - pypi_htpasswd.txt: - file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/pypi_htpasswd.txt diff --git a/monitor/sample.sh b/monitor/sample.sh deleted file mode 100755 index 7401c47..0000000 --- a/monitor/sample.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/sh -# Host-level network sampler for diagnosing the intermittent wheel-download -# connect timeouts (uv reports "client error (Connect): operation timed out"). -# -# Runs as a docker-compose sidecar with network_mode: host, so the /proc/net -# counters below reflect the *host* TCP stack -- where inbound :443 SYNs are -# accepted and NATed to the caddy container -- rather than the sidecar's own -# namespace. It only reads /proc, so no extra tooling is needed. -# -# Emits one TSV row per interval. The failure is a connect timeout, which -# never reaches Caddy and so leaves no trace in Caddy's access log; these -# host counters are how we tell *where* the connection is being lost: -# -# * listen_overflows / listen_drops / reqq_full_drop climbing, or syn_recv -# spiking, or conntrack near conntrack_max, or load1 high -> the loss is -# on THIS host (accept-queue overflow / table exhaustion / CPU stall), and -# we tune the host or Caddy accordingly. -# * all of the above flat and low while CI reports failures -> the SYNs are -# being dropped UPSTREAM (OpenStack floating-IP NAT / network path) and -# never reached us; the fix is an infra/network escalation, not the server. -# -# Counters from /proc/net/{snmp,netstat} are cumulative since boot -- diff -# adjacent rows (by epoch) during analysis to get per-interval rates. -# -# Caveat: with Docker's userland-proxy enabled (the default), host:443 has a -# docker-proxy listener whose accept-queue drops show up here as -# listen_drops/overflows; with it disabled (pure iptables DNAT) accept-queue -# pressure instead lands in the caddy container's namespace, so cross-check -# against Caddy's caddy_http_requests_in_flight metric when interpreting. - -set -u - -INTERVAL="${SAMPLE_INTERVAL:-15}" -OUT="${SAMPLE_OUT:-/var/log/netmon/net-samples.tsv}" -MAX_BYTES="${SAMPLE_MAX_BYTES:-104857600}" # rotate at 100 MiB, keep one prior - -mkdir -p "$(dirname "$OUT")" - -# Pull one named field out of /proc/net/snmp or /proc/net/netstat. These files -# store, per protocol label, a header row of field names followed by a values -# row; map names->columns from the header, then print the matching value. -field() { - # $1=file $2=label (e.g. Tcp, TcpExt) $3=field name - awk -v L="$2:" -v F="$3" ' - $1==L { - if (!(L in seen)) { for (i=2;i<=NF;i++) col[$i]=i; seen[L]=1; next } - print $(col[F]) - }' "$1" -} - -if [ ! -f "$OUT" ]; then - printf 'epoch\tiso\tload1\tconntrack\tconntrack_max\tcurr_estab\tsyn_recv\tactive_opens\tpassive_opens\tattempt_fails\tretrans_segs\tout_rsts\tlisten_overflows\tlisten_drops\treqq_full_drop\tsyncookies_sent\ttcp_timeouts\tsyn_retrans\n' > "$OUT" -fi - -while :; do - epoch=$(date +%s) - iso=$(date -u +%Y-%m-%dT%H:%M:%SZ) - load1=$(cut -d' ' -f1 /proc/loadavg) - ct=$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null || echo -1) - ctmax=$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null || echo -1) - - curr_estab=$(field /proc/net/snmp Tcp CurrEstab) - active_opens=$(field /proc/net/snmp Tcp ActiveOpens) - passive_opens=$(field /proc/net/snmp Tcp PassiveOpens) - attempt_fails=$(field /proc/net/snmp Tcp AttemptFails) - retrans_segs=$(field /proc/net/snmp Tcp RetransSegs) - out_rsts=$(field /proc/net/snmp Tcp OutRsts) - - listen_overflows=$(field /proc/net/netstat TcpExt ListenOverflows) - listen_drops=$(field /proc/net/netstat TcpExt ListenDrops) - reqq_full_drop=$(field /proc/net/netstat TcpExt TCPReqQFullDrop) - syncookies_sent=$(field /proc/net/netstat TcpExt SyncookiesSent) - tcp_timeouts=$(field /proc/net/netstat TcpExt TCPTimeouts) - syn_retrans=$(field /proc/net/netstat TcpExt TCPSynRetrans) - - # SYN_RECV (TCP state 0x03) across IPv4+IPv6: half-open / accept-queue depth. - syn_recv=$(cat /proc/net/tcp /proc/net/tcp6 2>/dev/null | awk '$4=="03"' | wc -l | tr -d ' ') - - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "$epoch" "$iso" "$load1" "$ct" "$ctmax" "$curr_estab" "$syn_recv" \ - "$active_opens" "$passive_opens" "$attempt_fails" "$retrans_segs" "$out_rsts" \ - "$listen_overflows" "$listen_drops" "$reqq_full_drop" "$syncookies_sent" \ - "$tcp_timeouts" "$syn_retrans" >> "$OUT" - - sz=$(wc -c < "$OUT" 2>/dev/null || echo 0) - if [ "$sz" -gt "$MAX_BYTES" ]; then - mv "$OUT" "$OUT.1" - fi - - sleep "$INTERVAL" -done