diff --git a/.github/workflows/probe-connection.yml b/.github/workflows/probe-connection.yml deleted file mode 100644 index bfaab2f..0000000 --- a/.github/workflows/probe-connection.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: probe -on: workflow_dispatch -jobs: - reach: - runs-on: ubuntu-slim - steps: - - run: | - echo "egress IP:"; curl -sS https://api.ipify.org; echo - echo "--- connect test ---" - curl -4 -v --connect-timeout 20 https://pypi.halide-lang.org/ 2>&1 | tail -25 diff --git a/README.md b/README.md index adbaf26..4373297 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,24 @@ $ uv sync --all-packages # Master configuration -The master is deployed via [Docker Compose][dc], which manages five services: -a PostgreSQL database, the Buildbot master, a [pypiserver] instance for hosting -`halide-llvm` wheels, a cleanup sidecar that evicts old development wheels, and -a [Caddy] reverse proxy with automatic HTTPS. +The master is deployed via [Docker Compose][dc], which manages three services: +a PostgreSQL database, the Buildbot master, and a [Caddy] reverse proxy with +automatic HTTPS. + +`halide-llvm` (and other) wheels used to be hosted here via a bundled +[pypiserver] instance at `pypi.halide-lang.org`, fronted by this same Caddy. +That's gone: the MIT-hosted VM this appliance runs on turned out to be subject +to intermittent border-security IP quarantines outside anyone's control +(shared cloud egress IPs occasionally get flagged for unrelated scanning +activity from other tenants, with no exception mechanism available). Package +hosting has moved to [halide/pypi] (GitHub Releases + Pages), which routes CI +traffic off that network path entirely. `pypi.halide-lang.org` now points +there via DNS; this repo no longer serves it. ## Secrets -Six secrets control authentication with external users and servers. These will -need to be determined before starting up a new master. +Five secrets control authentication with external users and servers. These +will need to be determined before starting up a new master. 1. Obtain a [GitHub personal access token](https://github.com/settings/tokens) with at least the `repo` scope enabled (other scopes that are not currently @@ -32,8 +41,6 @@ need to be determined before starting up a new master. This is only needed when using PostgreSQL (i.e., when `HALIDE_BB_MASTER_DB_URL` contains `{DB_PASSWORD}`). The default SQLite backend does not require it. -6. Generate a password for uploading packages to the PyPI server. Call this - `PYPI_PASSWORD`. A convenient command for generating a secure secret is `openssl rand -hex 20`. @@ -45,7 +52,6 @@ $ echo "$WORKER_SECRET" > secrets/halide_bb_pass.txt $ echo "$WEBHOOK_SECRET" > secrets/webhook_token.txt $ echo "$WWW_PASSWORD" > secrets/buildbot_www_pass.txt $ echo "$DB_PASSWORD" > secrets/db_password.txt -$ docker run --rm httpd:2-alpine htpasswd -Bbn upload "$PYPI_PASSWORD" > secrets/pypi_htpasswd.txt ``` ## GitHub configuration @@ -64,15 +70,6 @@ the correct ones: ## Starting the master -Optionally choose a directory to hold PyPI packages (the default is -`./data/packages`): - -```console -$ export HALIDE_BB_PYPI_PACKAGES_DIR=$HOME/wheels -``` - -Then start all services: - ```console $ docker compose up -d --build ``` @@ -145,5 +142,6 @@ the worker to start automatically are provided under `worker/`: [Caddy]: https://caddyserver.com [dc]: https://docs.docker.com/compose/ +[halide/pypi]: https://github.com/halide/pypi [pypiserver]: https://github.com/pypiserver/pypiserver [uv]: https://docs.astral.sh/uv diff --git a/caddy/Caddyfile b/caddy/Caddyfile index a4825ad..ea5e91f 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -7,49 +7,6 @@ metrics } -pypi.halide-lang.org { - # Access log for every request that reaches Caddy. This is the key signal - # for the intermittent wheel-download failures: a truncated download shows - # up as status 200 with a `size` smaller than the wheel's content-length, - # whereas a client-side connect timeout never appears here at all (which - # would point at the network path rather than the server). Written to a - # dedicated volume so it is not interleaved with Caddy's TLS/ACME logs and - # is not rolled off by Docker's log driver. - log { - output file /var/log/caddy/access.log { - roll_size 100MiB - roll_keep 20 - roll_keep_for 2160h - } - format json - } - - # Serve the (large, static) wheel files directly from disk so that - # downloads don't tie up pypiserver's small Waitress thread pool. Only - # the generated indexes and uploads (POST) still hit the Python server. - # - # The bare /packages/ path is the index listing that pypiserver renders, - # so it must keep going to the backend; only requests for actual files - # (/packages/) are served statically. handle blocks are mutually - # exclusive and evaluated in order, so the exact-match listing wins first. - handle /packages/ { - reverse_proxy pypi:8080 - } - - handle_path /packages/* { - root * /wheels - file_server - } - - reverse_proxy pypi:8080 - - header { - Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" - X-Frame-Options DENY - X-Content-Type-Options nosniff - } -} - buildbot.halide-lang.org { redir / /master/ temporary diff --git a/docker-compose.yml b/docker-compose.yml index 15e36a7..9ec63f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,17 +39,6 @@ services: driver: local restart: unless-stopped - pypi: - image: pypiserver/pypiserver:v2.4.0 - volumes: - - ${HALIDE_BB_PYPI_PACKAGES_DIR:-./data/packages}:/data/packages - secrets: - - pypi_htpasswd.txt - command: run -P /run/secrets/pypi_htpasswd.txt -a update --backend cached-dir /data/packages - logging: - driver: local - restart: unless-stopped - caddy: image: caddy:2-alpine # --watch reloads the Caddyfile on change. It works only because the @@ -65,27 +54,8 @@ services: volumes: - ./caddy:/etc/caddy:ro - ./www:/srv:ro - - ${HALIDE_BB_PYPI_PACKAGES_DIR:-./data/packages}:/wheels:ro - caddy-data:/data - caddy-config:/config - - caddy-logs:/var/log/caddy - logging: - driver: local - restart: unless-stopped - - # Host-level TCP/network sampler for diagnosing the intermittent - # wheel-download connect timeouts. network_mode: host so the /proc/net - # counters it reads reflect the host stack that terminates inbound :443, - # not this container's own namespace. See monitor/sample.sh. - netmon: - image: alpine:3.20 - network_mode: host - command: sh /monitor/sample.sh - volumes: - - ./monitor:/monitor:ro - - netmon-logs:/var/log/netmon - environment: - SAMPLE_INTERVAL: "15" logging: driver: local restart: unless-stopped @@ -93,8 +63,6 @@ services: volumes: caddy-data: caddy-config: - caddy-logs: - netmon-logs: db-data: secrets: @@ -108,5 +76,3 @@ secrets: file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/webhook_token.txt buildbot_www_pass.txt: file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/buildbot_www_pass.txt - pypi_htpasswd.txt: - file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/pypi_htpasswd.txt diff --git a/monitor/sample.sh b/monitor/sample.sh deleted file mode 100755 index 7401c47..0000000 --- a/monitor/sample.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/sh -# Host-level network sampler for diagnosing the intermittent wheel-download -# connect timeouts (uv reports "client error (Connect): operation timed out"). -# -# Runs as a docker-compose sidecar with network_mode: host, so the /proc/net -# counters below reflect the *host* TCP stack -- where inbound :443 SYNs are -# accepted and NATed to the caddy container -- rather than the sidecar's own -# namespace. It only reads /proc, so no extra tooling is needed. -# -# Emits one TSV row per interval. The failure is a connect timeout, which -# never reaches Caddy and so leaves no trace in Caddy's access log; these -# host counters are how we tell *where* the connection is being lost: -# -# * listen_overflows / listen_drops / reqq_full_drop climbing, or syn_recv -# spiking, or conntrack near conntrack_max, or load1 high -> the loss is -# on THIS host (accept-queue overflow / table exhaustion / CPU stall), and -# we tune the host or Caddy accordingly. -# * all of the above flat and low while CI reports failures -> the SYNs are -# being dropped UPSTREAM (OpenStack floating-IP NAT / network path) and -# never reached us; the fix is an infra/network escalation, not the server. -# -# Counters from /proc/net/{snmp,netstat} are cumulative since boot -- diff -# adjacent rows (by epoch) during analysis to get per-interval rates. -# -# Caveat: with Docker's userland-proxy enabled (the default), host:443 has a -# docker-proxy listener whose accept-queue drops show up here as -# listen_drops/overflows; with it disabled (pure iptables DNAT) accept-queue -# pressure instead lands in the caddy container's namespace, so cross-check -# against Caddy's caddy_http_requests_in_flight metric when interpreting. - -set -u - -INTERVAL="${SAMPLE_INTERVAL:-15}" -OUT="${SAMPLE_OUT:-/var/log/netmon/net-samples.tsv}" -MAX_BYTES="${SAMPLE_MAX_BYTES:-104857600}" # rotate at 100 MiB, keep one prior - -mkdir -p "$(dirname "$OUT")" - -# Pull one named field out of /proc/net/snmp or /proc/net/netstat. These files -# store, per protocol label, a header row of field names followed by a values -# row; map names->columns from the header, then print the matching value. -field() { - # $1=file $2=label (e.g. Tcp, TcpExt) $3=field name - awk -v L="$2:" -v F="$3" ' - $1==L { - if (!(L in seen)) { for (i=2;i<=NF;i++) col[$i]=i; seen[L]=1; next } - print $(col[F]) - }' "$1" -} - -if [ ! -f "$OUT" ]; then - printf 'epoch\tiso\tload1\tconntrack\tconntrack_max\tcurr_estab\tsyn_recv\tactive_opens\tpassive_opens\tattempt_fails\tretrans_segs\tout_rsts\tlisten_overflows\tlisten_drops\treqq_full_drop\tsyncookies_sent\ttcp_timeouts\tsyn_retrans\n' > "$OUT" -fi - -while :; do - epoch=$(date +%s) - iso=$(date -u +%Y-%m-%dT%H:%M:%SZ) - load1=$(cut -d' ' -f1 /proc/loadavg) - ct=$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null || echo -1) - ctmax=$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null || echo -1) - - curr_estab=$(field /proc/net/snmp Tcp CurrEstab) - active_opens=$(field /proc/net/snmp Tcp ActiveOpens) - passive_opens=$(field /proc/net/snmp Tcp PassiveOpens) - attempt_fails=$(field /proc/net/snmp Tcp AttemptFails) - retrans_segs=$(field /proc/net/snmp Tcp RetransSegs) - out_rsts=$(field /proc/net/snmp Tcp OutRsts) - - listen_overflows=$(field /proc/net/netstat TcpExt ListenOverflows) - listen_drops=$(field /proc/net/netstat TcpExt ListenDrops) - reqq_full_drop=$(field /proc/net/netstat TcpExt TCPReqQFullDrop) - syncookies_sent=$(field /proc/net/netstat TcpExt SyncookiesSent) - tcp_timeouts=$(field /proc/net/netstat TcpExt TCPTimeouts) - syn_retrans=$(field /proc/net/netstat TcpExt TCPSynRetrans) - - # SYN_RECV (TCP state 0x03) across IPv4+IPv6: half-open / accept-queue depth. - syn_recv=$(cat /proc/net/tcp /proc/net/tcp6 2>/dev/null | awk '$4=="03"' | wc -l | tr -d ' ') - - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "$epoch" "$iso" "$load1" "$ct" "$ctmax" "$curr_estab" "$syn_recv" \ - "$active_opens" "$passive_opens" "$attempt_fails" "$retrans_segs" "$out_rsts" \ - "$listen_overflows" "$listen_drops" "$reqq_full_drop" "$syncookies_sent" \ - "$tcp_timeouts" "$syn_retrans" >> "$OUT" - - sz=$(wc -c < "$OUT" 2>/dev/null || echo 0) - if [ "$sz" -gt "$MAX_BYTES" ]; then - mv "$OUT" "$OUT.1" - fi - - sleep "$INTERVAL" -done