Drop pypi server, diagnostic instrumentation, and connectivity probe

README.md

@@ -8,15 +8,24 @@ $ uv sync --all-packages
 
 # Master configuration
 
-The master is deployed via [Docker Compose][dc], which manages five services:
-a PostgreSQL database, the Buildbot master, a [pypiserver] instance for hosting
-`halide-llvm` wheels, a cleanup sidecar that evicts old development wheels, and
-a [Caddy] reverse proxy with automatic HTTPS.
+The master is deployed via [Docker Compose][dc], which manages three services:
+a PostgreSQL database, the Buildbot master, and a [Caddy] reverse proxy with
+automatic HTTPS.
+
+`halide-llvm` (and other) wheels used to be hosted here via a bundled
+[pypiserver] instance at `pypi.halide-lang.org`, fronted by this same Caddy.
+That's gone: the MIT-hosted VM this appliance runs on turned out to be subject
+to intermittent border-security IP quarantines outside anyone's control
+(shared cloud egress IPs occasionally get flagged for unrelated scanning
+activity from other tenants, with no exception mechanism available). Package
+hosting has moved to [halide/pypi] (GitHub Releases + Pages), which routes CI
+traffic off that network path entirely. `pypi.halide-lang.org` now points
+there via DNS; this repo no longer serves it.
 
 ## Secrets
 
-Six secrets control authentication with external users and servers. These will
-need to be determined before starting up a new master.
+Five secrets control authentication with external users and servers. These
+will need to be determined before starting up a new master.
 
 1. Obtain a [GitHub personal access token](https://github.com/settings/tokens)
    with at least the `repo` scope enabled (other scopes that are not currently
@@ -32,8 +41,6 @@ need to be determined before starting up a new master.
    This is only needed when using PostgreSQL (i.e., when
    `HALIDE_BB_MASTER_DB_URL` contains `{DB_PASSWORD}`). The default SQLite
    backend does not require it.
-6. Generate a password for uploading packages to the PyPI server. Call this
-   `PYPI_PASSWORD`.
 
 A convenient command for generating a secure secret is `openssl rand -hex 20`.
 
@@ -45,7 +52,6 @@ $ echo "$WORKER_SECRET" > secrets/halide_bb_pass.txt
 $ echo "$WEBHOOK_SECRET" > secrets/webhook_token.txt
 $ echo "$WWW_PASSWORD" > secrets/buildbot_www_pass.txt
 $ echo "$DB_PASSWORD" > secrets/db_password.txt
-$ docker run --rm httpd:2-alpine htpasswd -Bbn upload "$PYPI_PASSWORD" > secrets/pypi_htpasswd.txt
 ```
 
 ## GitHub configuration
@@ -64,15 +70,6 @@ the correct ones:
 
 ## Starting the master
 
-Optionally choose a directory to hold PyPI packages (the default is
-`./data/packages`):
-
-```console
-$ export HALIDE_BB_PYPI_PACKAGES_DIR=$HOME/wheels
-```
-
-Then start all services:
-
 ```console
 $ docker compose up -d --build
 ```
@@ -145,5 +142,6 @@ the worker to start automatically are provided under `worker/`:
 
 [Caddy]: https://caddyserver.com
 [dc]: https://docs.docker.com/compose/
+[halide/pypi]: https://github.com/halide/pypi
 [pypiserver]: https://github.com/pypiserver/pypiserver
 [uv]: https://docs.astral.sh/uv

caddy/Caddyfile

@@ -7,49 +7,6 @@
 	metrics
 }
 
-pypi.halide-lang.org {
-	# Access log for every request that reaches Caddy. This is the key signal
-	# for the intermittent wheel-download failures: a truncated download shows
-	# up as status 200 with a `size` smaller than the wheel's content-length,
-	# whereas a client-side connect timeout never appears here at all (which
-	# would point at the network path rather than the server). Written to a
-	# dedicated volume so it is not interleaved with Caddy's TLS/ACME logs and
-	# is not rolled off by Docker's log driver.
-	log {
-		output file /var/log/caddy/access.log {
-			roll_size 100MiB
-			roll_keep 20
-			roll_keep_for 2160h
-		}
-		format json
-	}
-
-	# Serve the (large, static) wheel files directly from disk so that
-	# downloads don't tie up pypiserver's small Waitress thread pool. Only
-	# the generated indexes and uploads (POST) still hit the Python server.
-	#
-	# The bare /packages/ path is the index listing that pypiserver renders,
-	# so it must keep going to the backend; only requests for actual files
-	# (/packages/<name>) are served statically. handle blocks are mutually
-	# exclusive and evaluated in order, so the exact-match listing wins first.
-	handle /packages/ {
-		reverse_proxy pypi:8080
-	}
-
-	handle_path /packages/* {
-		root * /wheels
-		file_server
-	}
-
-	reverse_proxy pypi:8080
-
-	header {
-		Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"
-		X-Frame-Options DENY
-		X-Content-Type-Options nosniff
-	}
-}
-
 buildbot.halide-lang.org {
 	redir / /master/ temporary
 

docker-compose.yml

@@ -39,17 +39,6 @@ services:
       driver: local
     restart: unless-stopped
 
-  pypi:
-    image: pypiserver/pypiserver:v2.4.0
-    volumes:
-      - ${HALIDE_BB_PYPI_PACKAGES_DIR:-./data/packages}:/data/packages
-    secrets:
-      - pypi_htpasswd.txt
-    command: run -P /run/secrets/pypi_htpasswd.txt -a update --backend cached-dir /data/packages
-    logging:
-      driver: local
-    restart: unless-stopped
-
   caddy:
     image: caddy:2-alpine
     # --watch reloads the Caddyfile on change. It works only because the
@@ -65,36 +54,15 @@ services:
     volumes:
       - ./caddy:/etc/caddy:ro
       - ./www:/srv:ro
-      - ${HALIDE_BB_PYPI_PACKAGES_DIR:-./data/packages}:/wheels:ro
       - caddy-data:/data
       - caddy-config:/config
-      - caddy-logs:/var/log/caddy
-    logging:
-      driver: local
-    restart: unless-stopped
-
-  # Host-level TCP/network sampler for diagnosing the intermittent
-  # wheel-download connect timeouts. network_mode: host so the /proc/net
-  # counters it reads reflect the host stack that terminates inbound :443,
-  # not this container's own namespace. See monitor/sample.sh.
-  netmon:
-    image: alpine:3.20
-    network_mode: host
-    command: sh /monitor/sample.sh
-    volumes:
-      - ./monitor:/monitor:ro
-      - netmon-logs:/var/log/netmon
-    environment:
-      SAMPLE_INTERVAL: "15"
     logging:
       driver: local
     restart: unless-stopped
 
 volumes:
   caddy-data:
   caddy-config:
-  caddy-logs:
-  netmon-logs:
   db-data:
 
 secrets:
@@ -108,5 +76,3 @@ secrets:
     file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/webhook_token.txt
   buildbot_www_pass.txt:
     file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/buildbot_www_pass.txt
-  pypi_htpasswd.txt:
-    file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/pypi_htpasswd.txt