From 7246ef4ef63b4da6001fdd1d24ced6afc8184e68 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Mon, 20 Jul 2026 21:16:19 +0000 Subject: [PATCH 1/9] feat: add opt-in rootless Docker + kind support to workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an enable_docker workspace parameter that runs a rootless Docker daemon inside the (still unprivileged, no-socket-mount) workspace pod, so k8s/gitops work — building images, running kind clusters for chainsaw tests — can happen in-workspace. Image: rootless-docker plumbing baked into system-base (uidmap, slirp4netns, fuse-overlayfs, iptables, docker-ce from Docker's apt repo for the dockerd-rootless helpers); subuid/subgid guaranteed for coder. kind itself comes from the operator's dotfiles (aqua), not the image. Template: enable_docker gates a docker-data PVC (survives stop/start, not start_count-gated), an XDG_RUNTIME_DIR emptyDir, DOCKER_HOST, and a gated rootless dockerd bring-up in the agent startup script. The pod carries a com.coder.workspace.docker-enabled marker label; a Kyverno policy (apps repo) injects spec.hostUsers=false since the kubernetes provider 3.2.1 has no host_users field yet. Co-Authored-By: Claude --- images/homelab-workspace/Dockerfile | 53 +++++++++++++ .../homelab-workspace/deployment.tf | 74 ++++++++++++++++++- templates/kubernetes/homelab-workspace/env.tf | 8 ++ .../homelab-workspace/parameters.tf | 11 +++ .../homelab-workspace/script-agent-startup.sh | 37 ++++++++++ 5 files changed, 182 insertions(+), 1 deletion(-) diff --git a/images/homelab-workspace/Dockerfile b/images/homelab-workspace/Dockerfile index 04acfc3b..6379369f 100644 --- a/images/homelab-workspace/Dockerfile +++ b/images/homelab-workspace/Dockerfile @@ -113,6 +113,50 @@ RUN --mount=type=cache,target=/var/cache/apt,id=cache-apt-${TARGETARCH},sharing= # Clean pycache created during apt-get install (as apt stills retains some crud in spite of PYTHONPYCACHEPREFIX) find /usr -name __pycache__ -exec rm -rf {} + +# ======================================================================================================== +# Rootless Docker support (the runtime kind/Kubernetes-in-Docker relies on; kind itself is installed by the +# user via aqua in their dotfiles, not baked into the image). +# Kept as a self-contained block (separate from the general toolset above) so it can later be lifted into a +# dedicated variant image without untangling it from the base package list. +# - uidmap: provides newuidmap/newgidmap (setuid-root) needed to map subordinate uid/gid ranges +# - slirp4netns: userspace network stack for rootless dockerd (no CAP_NET_ADMIN required) +# - fuse-overlayfs: fallback storage driver; native overlay2 (userns + kernel>=5.11) is the primary path, +# used when /dev/fuse is unavailable (the unprivileged pod has no /dev/fuse) +# - iptables: required by dockerd/kind for bridge + port-mapping rules +# docker-ce (from Docker's official apt repo, not Ubuntu's docker.io) is required because it ships +# dockerd-rootless.sh and dockerd-rootless-setuptool.sh, which the Ubuntu package does not. +RUN --mount=type=cache,target=/var/cache/apt,id=cache-apt-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/var/cache/debconf,id=cache-debconf-${TARGETARCH},sharing=locked \ + --mount=type=cache,target=/var/lib/apt,id=lib-apt-${TARGETARCH},sharing=locked \ + --mount=type=tmpfs,target=/var/cache/python \ + --mount=type=tmpfs,target=/tmp \ + --mount=type=tmpfs,target=/var/log \ + --mount=type=tmpfs,target=/var/tmp \ + # rootless-docker runtime dependencies (from Ubuntu repos) + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -yq \ + fuse-overlayfs \ + iptables \ + slirp4netns \ + uidmap \ + && \ + # add Docker's official apt repository + signing key (arch matched to the build platform) + install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + echo "deb [arch=${TARGETARCH} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "${VERSION_CODENAME}") stable" \ + > /etc/apt/sources.list.d/docker.list && \ + # Docker engine + CLI + buildx + containerd (docker-ce ships dockerd-rootless*.sh helpers) + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -yq \ + containerd.io \ + docker-buildx-plugin \ + docker-ce \ + docker-ce-cli \ + && \ + # Clean pycache created during apt-get install (as apt stills retains some crud in spite of PYTHONPYCACHEPREFIX) + find /usr -name __pycache__ -exec rm -rf {} + + # ======================================================================================================== FROM system-base SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -142,5 +186,14 @@ RUN groupadd --gid "${CODER_GID}" coder && \ # package installation is complete, users of resulting image need not cache downloaded packages rm -f /etc/apt/apt.conf.d/keep-cache +# Guarantee subordinate uid/gid ranges for 'coder' so rootless dockerd can map users inside its userns. +# useradd does not reliably populate /etc/subuid+/etc/subgid at build time, so ensure them idempotently +# (append only if no entry for coder exists; never clobber ranges already present at runtime). +# newuidmap/newgidmap must stay setuid-root for the mapping to work - verify, and do NOT strip the bits. +RUN touch /etc/subuid /etc/subgid && \ + grep -q '^coder:' /etc/subuid || echo 'coder:100000:65536' >> /etc/subuid && \ + grep -q '^coder:' /etc/subgid || echo 'coder:100000:65536' >> /etc/subgid && \ + test -u /usr/bin/newuidmap && test -u /usr/bin/newgidmap + USER coder WORKDIR /home/coder diff --git a/templates/kubernetes/homelab-workspace/deployment.tf b/templates/kubernetes/homelab-workspace/deployment.tf index f4e22cd3..de4be870 100644 --- a/templates/kubernetes/homelab-workspace/deployment.tf +++ b/templates/kubernetes/homelab-workspace/deployment.tf @@ -1,3 +1,11 @@ +locals { + # Marker label consumed by a Kyverno policy (separate repo) that injects + # `hostUsers: false` onto the pod — the hashicorp/kubernetes provider has no + # host_users attribute, so this label is the only way to request it. Absent + # (not "false") when docker is disabled so the policy simply doesn't match. + docker_labels = data.coder_parameter.enable_docker.value ? { "com.coder.workspace.docker-enabled" = "true" } : {} +} + resource "kubernetes_deployment_v1" "deployment" { count = data.coder_workspace.me.start_count @@ -22,7 +30,7 @@ resource "kubernetes_deployment_v1" "deployment" { template { metadata { - labels = merge(local.common_labels, local.pod_labels) + labels = merge(local.common_labels, local.pod_labels, local.docker_labels) } spec { dynamic "affinity" { @@ -86,6 +94,10 @@ resource "kubernetes_deployment_v1" "deployment" { name = "CODER_AGENT_TOKEN" value = coder_agent.main.token } + env { + name = "ENABLE_DOCKER" + value = tostring(data.coder_parameter.enable_docker.value) + } liveness_probe { exec { command = ["/bin/sh", "-c", "pgrep -f \"coder agent\" || exit 1"] @@ -151,6 +163,20 @@ resource "kubernetes_deployment_v1" "deployment" { name = "system" sub_path = "var" } + dynamic "volume_mount" { + for_each = data.coder_parameter.enable_docker.value ? toset(["docker-data"]) : [] + content { + name = "docker-data" + mount_path = "${local.home_directory}/.local/share/docker" + } + } + dynamic "volume_mount" { + for_each = data.coder_parameter.enable_docker.value ? toset(["xdg-runtime"]) : [] + content { + name = "xdg-runtime" + mount_path = "/run/user/10001" + } + } } enable_service_links = false hostname = lower(replace(data.coder_workspace.me.name, "/[^a-zA-Z0-9]/", "-")) @@ -197,6 +223,52 @@ resource "kubernetes_deployment_v1" "deployment" { size_limit = "10Gi" } } + dynamic "volume" { + for_each = data.coder_parameter.enable_docker.value ? toset(["docker-data"]) : [] + content { + name = "docker-data" + persistent_volume_claim { + claim_name = kubernetes_persistent_volume_claim_v1.docker_data[0].metadata[0].name + read_only = false + } + } + } + dynamic "volume" { + for_each = data.coder_parameter.enable_docker.value ? toset(["xdg-runtime"]) : [] + content { + name = "xdg-runtime" + empty_dir {} + } + } + } + } + } +} + +# Persistent storage for the rootless Docker daemon's image/layer data. +# +# Gated on enable_docker but deliberately NOT on start_count: like the external +# `coder-workspace-home` PVC, this must survive a workspace STOP (deployment +# scales to 0). Tying it to start_count would delete the volume — and every +# pulled image and built layer — on every stop. It is instead destroyed only on +# workspace DELETE or when docker is disabled. +resource "kubernetes_persistent_volume_claim_v1" "docker_data" { + count = data.coder_parameter.enable_docker.value ? 1 : 0 + + wait_until_bound = false + + metadata { + name = "coder-${data.coder_workspace.me.id}-docker" + namespace = "coder" + labels = merge(local.common_labels, local.pod_labels) + } + + spec { + access_modes = ["ReadWriteOnce"] + storage_class_name = "sc-longhorn-local-non-replicated" + resources { + requests = { + storage = "40Gi" } } } diff --git a/templates/kubernetes/homelab-workspace/env.tf b/templates/kubernetes/homelab-workspace/env.tf index 4b7449db..84b029ac 100644 --- a/templates/kubernetes/homelab-workspace/env.tf +++ b/templates/kubernetes/homelab-workspace/env.tf @@ -3,3 +3,11 @@ resource "coder_env" "welcome_message" { name = "HOMEBREW_PREFIX" value = local.homebrew_directory } + +resource "coder_env" "docker_host" { + count = data.coder_parameter.enable_docker.value ? 1 : 0 + + agent_id = coder_agent.main.id + name = "DOCKER_HOST" + value = "unix:///run/user/10001/docker.sock" +} diff --git a/templates/kubernetes/homelab-workspace/parameters.tf b/templates/kubernetes/homelab-workspace/parameters.tf index c3215313..d531e419 100644 --- a/templates/kubernetes/homelab-workspace/parameters.tf +++ b/templates/kubernetes/homelab-workspace/parameters.tf @@ -61,6 +61,17 @@ data "coder_parameter" "system_packages" { type = "list(string)" } +data "coder_parameter" "enable_docker" { + name = "enable_docker" + + default = false + display_name = "Enable Docker" + description = "Run a rootless Docker daemon inside the workspace" + icon = "/icon/docker.svg" + mutable = true + type = "bool" +} + locals { validated_system_packages = (data.coder_parameter.system_packages.value != "") ? [ diff --git a/templates/kubernetes/homelab-workspace/script-agent-startup.sh b/templates/kubernetes/homelab-workspace/script-agent-startup.sh index 957175b2..7db66a63 100644 --- a/templates/kubernetes/homelab-workspace/script-agent-startup.sh +++ b/templates/kubernetes/homelab-workspace/script-agent-startup.sh @@ -2,6 +2,38 @@ set -eo pipefail +setup_docker() { + export XDG_RUNTIME_DIR="/run/user/10001" + export DOCKER_HOST="unix://${XDG_RUNTIME_DIR}/docker.sock" + # dockerd-rootless.sh and dockerd-rootless-setuptool.sh ship in /usr/bin + export PATH="/usr/bin:${PATH}" + + echo 'Ensuring XDG_RUNTIME_DIR exists...' + mkdir -p "${XDG_RUNTIME_DIR}" + chmod 700 "${XDG_RUNTIME_DIR}" + + echo 'Installing rootless docker (idempotent; tolerate re-run)...' + dockerd-rootless-setuptool.sh install 2>&1 | sed -E -n 's|^| |p' || true + + echo 'Starting rootless dockerd...' + # Storage driver is auto-detected by dockerd with preference order + # overlay2 -> fuse-overlayfs -> vfs. When overlay2 is unavailable in this + # (userns) environment docker falls back on its own; pass `--storage-driver` + # to dockerd-rootless.sh only if a specific driver must be forced. + setsid dockerd-rootless.sh > "${HOME}/.local/share/docker/dockerd.log" 2>&1 & + + echo 'Waiting up to ~30s for docker to become ready...' + for _ in $(seq 1 30); do + if docker info > /dev/null 2>&1; then + echo 'Docker is ready.' + return 0 + fi + sleep 1 + done + echo "Docker did not become ready; see ${HOME}/.local/share/docker/dockerd.log" + return 1 +} + main() { if [[ ! -s ~/.bashrc ]]; then echo "Setting up starter bash rc scripts from /etc/skel..." @@ -10,6 +42,11 @@ main() { echo 'set -o allexport; source /etc/environment; set +o allexport' >> ~/.bashrc echo '------------------------------------------------------------' fi + if [[ "${ENABLE_DOCKER:-false}" == "true" ]]; then + echo "Setting up rootless Docker..." + # Non-fatal: a docker failure must not abort agent startup. + setup_docker 2>&1 | sed -E -n 's|^| |p' || true + fi echo 'Done' } From 8149e7aad9d235f741bfea220fe17e7a83e9100e Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Mon, 20 Jul 2026 21:24:35 +0000 Subject: [PATCH 2/9] docs: explain rootless Docker + kind design and testing New DESIGN-DIND.md covers why the workspace runs Docker/kind while staying unprivileged (rejected alternatives, the Kyverno-injected hostUsers indirection, storage/driver trade-offs, and the cluster-migration angle). DESIGN.md, README.md, CLAUDE.md, and TESTING.md gain brief pointers plus the enable_docker gotchas and the live verification recipe. Co-Authored-By: Claude --- CLAUDE.md | 1 + DESIGN-DIND.md | 120 +++++++++++++++++++++++++++++++++++++++++++++++++ DESIGN.md | 2 + README.md | 1 + TESTING.md | 11 +++++ 5 files changed, 135 insertions(+) create mode 100644 DESIGN-DIND.md diff --git a/CLAUDE.md b/CLAUDE.md index 8b5d6760..687a6d6d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,6 +80,7 @@ Things that look arbitrary in the code but are load-bearing (full reasoning in [ - The Dockerfile writes shared env vars to `/etc/environment` rather than using `ENV`, because `PATH` needs to be extended by a script running after the image is built, not fixed at build time. - `parameters.tf`'s `local.validated_*` regex allowlist is the only thing stopping `system_packages`/`preferred_nodes` from injecting shell metacharacters into the init container — any new list-type parameter must go through the same decode-then-validate step. - Adding a package/tool has three possible homes, and picking the wrong one is a real mistake, not a style choice — route by the rule in [DESIGN.md](DESIGN.md#where-the-workspace-environment-comes-from): universal + stable → image (`Dockerfile`); occasionally-needed + apt-only + too heavy to bake in → the template's `system_packages` parameter; personal, fast-moving, or not an apt package → the operator's dotfiles (a *different* repo — see below), never this one. +- The rootless-Docker feature (`enable_docker` parameter) spans this repo *and* the apps repo, and has non-obvious load-bearing details — read [DESIGN-DIND.md](DESIGN-DIND.md) before touching it. Key traps: (1) the `kubernetes` provider 3.2.1 has **no `host_users` field**, so `hostUsers: false` is set by a Kyverno policy in the apps repo keyed off the `com.coder.workspace.docker-enabled` marker label — renaming that label breaks the contract silently; (2) the docker-data PVC is gated on `enable_docker` but **not** on `start_count` (like the home PVC) so it survives workspace stop — gating it on `start_count` would wipe every pulled image on each stop; (3) docker's plumbing is baked into the image but its *behaviour* is parameter-gated, because the setuid `newuidmap`/`newgidmap` + subuid/subgid can only be set at build time as root; those setuid bits must survive the init container's `rsync` into the `system` volume. ## Neighbouring repos diff --git a/DESIGN-DIND.md b/DESIGN-DIND.md new file mode 100644 index 00000000..c2338bac --- /dev/null +++ b/DESIGN-DIND.md @@ -0,0 +1,120 @@ +# DESIGN-DIND.md — nested containers in the workspace + +Why the workspace can run Docker and `kind` while staying an unprivileged pod, and how the +pieces fit. This is a focused companion to [DESIGN.md](DESIGN.md); read that first for the +repo-wide intent and the image/template split. + +## Intent + +A lot of the work done in these workspaces is Kubernetes/GitOps: building container images and +standing up throwaway `kind` clusters (for example, to run another repo's `chainsaw` tests +locally, the same way CI does). That needs a container runtime *inside* the workspace. + +The constraint that shapes everything: **the workspace must stay unprivileged**. No `privileged` +container, no `CAP_SYS_ADMIN`, no host Docker socket mounted in. The workspace runs as a fixed +non-root UID and should not be a foothold onto the node. So "just run Docker" is exactly what we +can't do — the whole design is about getting a usable container runtime *without* relaxing that. + +## Options considered + +The obvious answers all fail the unprivileged/low-blast-radius bar in different ways: + +| Approach | Why not | +| --- | --- | +| Privileged Docker sidecar / mount the node's docker socket | Gives the workspace root on the node. Rejected outright. | +| **Sysbox** runtime | Solid isolation, but it's a **node-level** runtime install + `RuntimeClass` — a platform component with high blast radius, and its maintenance/OS-support story is shaky. | +| **Envbox** | Wraps Sysbox but requires a **privileged outer pod**. Rejects on the same privilege bar. | +| **Kata Containers** | Real VM isolation, but a node-level runtime + `RuntimeClass` dependency and needs KVM/nested-virt. High blast radius for a homelab. | +| **Rootless Docker/Podman inside the pod** | Runtime is **pod-local** — nothing installed on the node. This is the one that fits. | + +Within the rootless family, **Docker** was chosen over Podman/nerdctl for one practical reason: +the consuming repo's CI runs `kind` on its **default Docker provider**, and matching that keeps +"what I run locally" identical to "what CI runs" — no experimental-provider divergence. The cost +is that rootless `dockerd` is the fiddliest of the three to bring up in a bare pod (no user +systemd session); accepted deliberately for the local-==-CI payoff. + +### The strong-isolation piece: user namespaces + +Rootless alone still benefits enormously from a **user namespace** that maps the workspace's +in-container root to an unprivileged host UID — that's what makes a container breakout land as +"nobody" on the node rather than something with reach. Kubernetes exposes this as +`pod.spec.hostUsers: false` (GA on the cluster we target). + +The catch, and the reason this design has a moving part that looks odd: **the Terraform +`kubernetes` provider we're pinned to has no field to set `hostUsers`**. Rather than abandon the +typed `kubernetes_deployment_v1` resource (rewriting the whole pod as a raw manifest loses +validation and forces unrelated refactors), we set the field **out of band with a Kyverno mutate +policy** that lives with the platform, keyed off a marker label the template stamps only when the +user opts in. The template stays typed and simple; the platform grants the privilege-reducing +mapping. When the provider gains the field, the policy is deleted and the template sets it +directly — a one-line change, not a redesign. + +## How the pieces compose + +```mermaid +flowchart TB + subgraph image [Image · baked plumbing] + I["rootless dockerd + CLI
uidmap · slirp4netns · fuse-overlayfs
subuid/subgid for coder"] + end + subgraph template [Template · gated on enable_docker] + P["enable_docker parameter"] + M["marker label
com.coder.workspace.docker-enabled=true"] + V["docker-data PVC + XDG_RUNTIME_DIR
DOCKER_HOST env"] + S["agent startup:
start rootless dockerd"] + end + subgraph platform [Platform · apps repo] + K["Kyverno mutate policy
injects spec.hostUsers=false"] + end + Pod["Unprivileged workspace pod
+ user namespace"] + Kind["docker → kind clusters"] + + I --> Pod + P --> M --> Pod + P --> V --> Pod + P --> S --> Pod + M -. matched by .-> K -. mutates .-> Pod + Pod --> Kind +``` + +The division of labour: the **image** carries the universal, stable plumbing (baked so startup is +fast and reproducible — the [DESIGN.md](DESIGN.md#where-the-workspace-environment-comes-from) +layering rule); the **template** turns it on per-workspace and provides the pod-local storage and +runtime wiring; the **platform policy** supplies the one thing the template can't express. `kind` +itself is *not* baked into the image — it's day-to-day tooling and comes from the operator's +dotfiles (aqua), per the same layering rule. + +## Trade-offs and things to know + +- **Opt-in, not baseline.** The plumbing is baked into every image (it must be — the setuid + `newuidmap`/`newgidmap` and subuid/subgid ranges can only be set at build time as root), but the + *behaviour* (daemon start, the PVC, `hostUsers` mutation) is gated behind `enable_docker`. A + workspace that doesn't opt in is byte-for-byte the same runtime posture as before. +- **Storage is a dedicated PVC, and survives stop/start.** Docker's image/layer store and `kind` + node images are large and worth keeping between sessions, so they live on their own + `ReadWriteOnce` block PVC rather than the shared home volume or an `emptyDir`. Crucially it is + gated on `enable_docker` but **not** on `start_count`, mirroring the external home PVC — tying it + to `start_count` would wipe every pulled image on each workspace *stop*. +- **Storage driver.** With a user namespace and a modern kernel, native `overlay2` works and needs + no `/dev/fuse` (which the unprivileged pod lacks); `fuse-overlayfs` is the fallback, `vfs` the + last resort. The daemon auto-detects. +- **Degrades, doesn't break.** If the Kyverno policy is absent or fails open, the pod still runs — + just without the `hostUsers` mapping (weaker isolation). The marker label and the policy are a + cross-repo contract; neither should be renamed in isolation. +- **No nested cgroup resource limits.** Rootless dockerd in a pod with no user systemd session + runs without delegated cgroup limits; the workspace pod's own Kubernetes limits still cap + everything, which is fine for a single operator. + +## Future: migrating the cluster runtime + +This design is deliberately runtime-agnostic and should *ease* a future migration to a different +Kubernetes distro rather than block it: + +- The image and startup scripts are not tied to the current distro; they carry over unchanged. +- User namespaces (`hostUsers: false`) are a standard Kubernetes feature; the target distro + already enables the necessary kubelet/runtime support, so the same mechanism applies. +- The Kyverno policy is platform-level and portable as-is. +- The one thing to re-check at migration time is that the docker-data PVC's storage class exists on + the new cluster (or swap it for the equivalent). + +The `hostUsers`-via-policy indirection is the only piece expected to be temporary: once the +Terraform provider exposes the field, the template sets it directly and the policy retires. diff --git a/DESIGN.md b/DESIGN.md index 4a384204..979bd268 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -63,6 +63,8 @@ The rule that ties the layers together: a package or tool belongs in the *lowest **Unprivileged by default.** The workspace itself runs as an unprivileged, non-root, fixed-identity container. Anything that genuinely needs elevated privilege (installing packages, preparing shared volume state) is scoped to a narrow, short-lived setup step that runs before the workspace shell exists, not to something the workspace user can reach into. +**Nested containers without giving up that posture.** Running Docker and `kind` *inside* the workspace would normally mean a privileged pod or a node-level runtime — both rejected here. Instead an opt-in rootless-Docker setup keeps the pod unprivileged and socket-free, with user-namespace isolation supplied by a platform Kyverno policy. This is a large enough topic to have its own design doc — see [DESIGN-DIND.md](DESIGN-DIND.md), which also covers the cluster-runtime migration angle. + ## Outcomes targeted - One operator can keep dependencies current and ship template/image changes at low ongoing effort, without a fleet of environments to maintain. diff --git a/README.md b/README.md index bf3808ed..4b60c2c9 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This repo is the middle of a larger stack: the Coder control plane is deployed s ## Docs - **[DESIGN.md](DESIGN.md)** — why the template and image are built the way they are: trade-offs considered, decisions made, targeted outcomes. +- **[DESIGN-DIND.md](DESIGN-DIND.md)** — how the workspace runs Docker and `kind` while staying an unprivileged pod. - **[TESTING.md](TESTING.md)** — how to validate a change, including exercising it against the real cluster without touching production workspace data. - **[CLAUDE.md](CLAUDE.md)** — commands and conventions for working in this repo with Claude Code. - **[CHANGELOG.md](CHANGELOG.md)** — generated release history (semantic-release). diff --git a/TESTING.md b/TESTING.md index 00d369cd..c838e126 100644 --- a/TESTING.md +++ b/TESTING.md @@ -23,6 +23,17 @@ Both modes run the identical sequence of stages; only what each stage is permitt Because all three stages run for real in dry-run — just scoped away from production — a passing PR is a meaningful signal that a live release would also succeed, not a guess based on static checks alone. +## Exercising the rootless Docker feature + +The `enable_docker` feature (see [DESIGN-DIND.md](DESIGN-DIND.md)) can't be judged by lint alone — it needs a live workspace. After the dry-run pipeline builds the image and pushes the disposable template, provision a test workspace from that template with **`enable_docker = true`** and check, from a terminal in the workspace: + +- `docker info` — the daemon is up and rootless (`Cgroup Driver` / `rootless: true`), storage driver is `overlay2` (or the `fuse-overlayfs`/`vfs` fallback). +- `docker run --rm hello-world` — a container actually runs. +- `kind create cluster` (kind comes from dotfiles) then `kubectl get nodes` — a nested cluster comes up; tear it down with `kind delete cluster`. +- `kubectl get pod -o jsonpath='{.spec.hostUsers}'` (from a client with cluster access) returns `false` — confirming the platform Kyverno policy mutated the pod. If it's empty, the policy side isn't in effect and Docker is running in the weaker posture. + +The Kyverno policy half lives in the platform repo and has its own chainsaw coverage there (it asserts a marked pod is mutated and an unmarked one is left alone) — a template change here that renames the `com.coder.workspace.docker-enabled` marker must be matched there. + ## After merge Merging to `main` is what flips the pipeline into live mode — there's no separate promotion step afterward. The dry-run pass on the PR is the actual release gate. From 280317d6098ed19ce4b7141a2054adcde0aa1663 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Mon, 20 Jul 2026 22:03:24 +0000 Subject: [PATCH 3/9] fix: install docker-ce-rootless-extras for the rootless entrypoints The rootless dockerd daemon never started in a real workspace: dockerd-rootless.sh was absent. Those entrypoints ship in the separate docker-ce-rootless-extras package, which docker-ce does not pull in. Add it, and have the startup script fail with a clear message (rather than a cryptic setsid error) if the binary is ever missing again. Co-Authored-By: Claude --- images/homelab-workspace/Dockerfile | 9 ++++++--- .../kubernetes/homelab-workspace/script-agent-startup.sh | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/images/homelab-workspace/Dockerfile b/images/homelab-workspace/Dockerfile index 6379369f..78846a84 100644 --- a/images/homelab-workspace/Dockerfile +++ b/images/homelab-workspace/Dockerfile @@ -123,8 +123,8 @@ RUN --mount=type=cache,target=/var/cache/apt,id=cache-apt-${TARGETARCH},sharing= # - fuse-overlayfs: fallback storage driver; native overlay2 (userns + kernel>=5.11) is the primary path, # used when /dev/fuse is unavailable (the unprivileged pod has no /dev/fuse) # - iptables: required by dockerd/kind for bridge + port-mapping rules -# docker-ce (from Docker's official apt repo, not Ubuntu's docker.io) is required because it ships -# dockerd-rootless.sh and dockerd-rootless-setuptool.sh, which the Ubuntu package does not. +# We use Docker's official apt repo (not Ubuntu's docker.io) so the rootless entrypoints ship in the +# docker-ce-rootless-extras package (dockerd-rootless.sh / dockerd-rootless-setuptool.sh). RUN --mount=type=cache,target=/var/cache/apt,id=cache-apt-${TARGETARCH},sharing=locked \ --mount=type=cache,target=/var/cache/debconf,id=cache-debconf-${TARGETARCH},sharing=locked \ --mount=type=cache,target=/var/lib/apt,id=lib-apt-${TARGETARCH},sharing=locked \ @@ -146,13 +146,16 @@ RUN --mount=type=cache,target=/var/cache/apt,id=cache-apt-${TARGETARCH},sharing= chmod a+r /etc/apt/keyrings/docker.asc && \ echo "deb [arch=${TARGETARCH} signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "${VERSION_CODENAME}") stable" \ > /etc/apt/sources.list.d/docker.list && \ - # Docker engine + CLI + buildx + containerd (docker-ce ships dockerd-rootless*.sh helpers) + # Docker engine + CLI + buildx + containerd. docker-ce-rootless-extras is a + # SEPARATE package (not pulled in by docker-ce) that ships dockerd-rootless.sh + # and dockerd-rootless-setuptool.sh -- the rootless entrypoints this image needs. apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -yq \ containerd.io \ docker-buildx-plugin \ docker-ce \ docker-ce-cli \ + docker-ce-rootless-extras \ && \ # Clean pycache created during apt-get install (as apt stills retains some crud in spite of PYTHONPYCACHEPREFIX) find /usr -name __pycache__ -exec rm -rf {} + diff --git a/templates/kubernetes/homelab-workspace/script-agent-startup.sh b/templates/kubernetes/homelab-workspace/script-agent-startup.sh index 7db66a63..e7683d37 100644 --- a/templates/kubernetes/homelab-workspace/script-agent-startup.sh +++ b/templates/kubernetes/homelab-workspace/script-agent-startup.sh @@ -8,6 +8,11 @@ setup_docker() { # dockerd-rootless.sh and dockerd-rootless-setuptool.sh ship in /usr/bin export PATH="/usr/bin:${PATH}" + if ! command -v dockerd-rootless.sh > /dev/null 2>&1; then + echo 'dockerd-rootless.sh not found (docker-ce-rootless-extras missing from image); skipping docker setup.' + return 1 + fi + echo 'Ensuring XDG_RUNTIME_DIR exists...' mkdir -p "${XDG_RUNTIME_DIR}" chmod 700 "${XDG_RUNTIME_DIR}" From c7a3507efc29f5f4474240df942cbd7a57dc9866 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Mon, 20 Jul 2026 22:32:49 +0000 Subject: [PATCH 4/9] fix: size coder subuid range to fit inside the pod user namespace Rootless dockerd failed with 'newuidmap: write to uid_map failed: Operation not permitted' under hostUsers=false. The pod userns is only 65536 IDs wide (0-65535), but useradd had set coder:165536:65536 -- pointing outside it. Our previous append-if-missing never fired because useradd's entry was already there. Overwrite with coder:10002:55534 (10002..65535, above coder's UID 10001), which fits the namespace. Also make the XDG_RUNTIME_DIR chmod non-fatal: the emptyDir is root-owned but group-writable via fsGroup, so it is already usable and the chmod only produced a spurious error. Co-Authored-By: Claude --- images/homelab-workspace/Dockerfile | 16 +++++++++------- .../homelab-workspace/script-agent-startup.sh | 5 ++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/images/homelab-workspace/Dockerfile b/images/homelab-workspace/Dockerfile index 78846a84..c47a4e73 100644 --- a/images/homelab-workspace/Dockerfile +++ b/images/homelab-workspace/Dockerfile @@ -189,13 +189,15 @@ RUN groupadd --gid "${CODER_GID}" coder && \ # package installation is complete, users of resulting image need not cache downloaded packages rm -f /etc/apt/apt.conf.d/keep-cache -# Guarantee subordinate uid/gid ranges for 'coder' so rootless dockerd can map users inside its userns. -# useradd does not reliably populate /etc/subuid+/etc/subgid at build time, so ensure them idempotently -# (append only if no entry for coder exists; never clobber ranges already present at runtime). -# newuidmap/newgidmap must stay setuid-root for the mapping to work - verify, and do NOT strip the bits. -RUN touch /etc/subuid /etc/subgid && \ - grep -q '^coder:' /etc/subuid || echo 'coder:100000:65536' >> /etc/subuid && \ - grep -q '^coder:' /etc/subgid || echo 'coder:100000:65536' >> /etc/subgid && \ +# Subordinate uid/gid ranges for 'coder', sized to fit INSIDE a user-namespaced pod. +# When the workspace runs with hostUsers=false (see DESIGN-DIND.md) the pod's userns is only +# 65536 IDs wide (0-65535), and newuidmap maps subordinate IDs relative to that namespace. +# useradd's default range (coder:165536:65536) points outside it, so rootlesskit fails with +# "newuidmap: write to uid_map failed: Operation not permitted". We therefore OVERWRITE (not +# append) with a range that fits above coder's own UID 10001: 10002..65535 -> 55534 subordinate +# IDs, enough for nested rootless containers. newuidmap/newgidmap must stay setuid-root. +RUN echo 'coder:10002:55534' > /etc/subuid && \ + echo 'coder:10002:55534' > /etc/subgid && \ test -u /usr/bin/newuidmap && test -u /usr/bin/newgidmap USER coder diff --git a/templates/kubernetes/homelab-workspace/script-agent-startup.sh b/templates/kubernetes/homelab-workspace/script-agent-startup.sh index e7683d37..4dee0408 100644 --- a/templates/kubernetes/homelab-workspace/script-agent-startup.sh +++ b/templates/kubernetes/homelab-workspace/script-agent-startup.sh @@ -15,7 +15,10 @@ setup_docker() { echo 'Ensuring XDG_RUNTIME_DIR exists...' mkdir -p "${XDG_RUNTIME_DIR}" - chmod 700 "${XDG_RUNTIME_DIR}" + # The dir is an emptyDir mount owned by root but group-writable via fsGroup=10001, + # so it is already usable by coder; chmod would fail (coder is not the owner) and + # that is harmless -- keep it non-fatal rather than emitting a scary error. + chmod 700 "${XDG_RUNTIME_DIR}" 2> /dev/null || true echo 'Installing rootless docker (idempotent; tolerate re-run)...' dockerd-rootless-setuptool.sh install 2>&1 | sed -E -n 's|^| |p' || true From 524e529d349f8e3d149fdf89a8e0c3ebcc31e0fa Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Mon, 20 Jul 2026 22:49:52 +0000 Subject: [PATCH 5/9] fix: pull image Always in test_mode so branch-tag rebuilds are picked up test_mode workspaces run a mutable branch tag (branch-) that is rebuilt in place. With the default IfNotPresent policy a node that cached the tag never re-pulls, so template/image fixes silently test against a stale image. Set imagePullPolicy=Always in test_mode; released (immutable) tags keep IfNotPresent. Co-Authored-By: Claude --- templates/kubernetes/homelab-workspace/deployment.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/templates/kubernetes/homelab-workspace/deployment.tf b/templates/kubernetes/homelab-workspace/deployment.tf index de4be870..6ed6eb5a 100644 --- a/templates/kubernetes/homelab-workspace/deployment.tf +++ b/templates/kubernetes/homelab-workspace/deployment.tf @@ -55,6 +55,10 @@ resource "kubernetes_deployment_v1" "deployment" { name = "prepare-workspace" command = ["/bin/bash", "/prepare-workspace-script.sh"] image = var.workspace_image + # In test_mode the image is a mutable branch tag that gets rebuilt in place, + # so a cached layer must not shadow a fresh push -> Always. Released versions + # use immutable tags where IfNotPresent is correct (and avoids needless pulls). + image_pull_policy = var.test_mode ? "Always" : "IfNotPresent" env { name = "SYSTEM_PACKAGES" value = length(local.validated_system_packages) > 0 ? join(" ", local.validated_system_packages) : "NONE" @@ -90,6 +94,8 @@ resource "kubernetes_deployment_v1" "deployment" { name = "workspace" command = ["/bin/bash", "/workspace-init.sh"] image = var.workspace_image + # See init_container: Always in test_mode (mutable branch tag), else IfNotPresent. + image_pull_policy = var.test_mode ? "Always" : "IfNotPresent" env { name = "CODER_AGENT_TOKEN" value = coder_agent.main.token From 9d8074a8a93c5a983cc95b4026776001ef0cf357 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Mon, 20 Jul 2026 23:57:00 +0000 Subject: [PATCH 6/9] feat: run workspace as namespaced-root with rootful dockerd Rootless-in-userns hit an unavoidable wall: allowPrivilegeEscalation=false sets no_new_privs, which neuters setuid newuidmap, so rootlesskit cannot set up its multi-UID map (EPERM) even with an in-range subuid. hostUsers=false already gives us safe namespaced root (root in the pod maps to an unprivileged host uid), so run the container as root and use a plain rootful dockerd instead -- no newuidmap, no subuid, no rootless plumbing. - New script-workspace-entrypoint.sh (runs as root): does the apt/homebrew prep (absorbed from the old init container), starts dockerd (non-fatal), then drops to the coder user via setpriv and execs the coder agent. The coder user keeps its 10001 identity and role. - deployment.tf: drop the init container and the whole system-emptyDir + /usr /etc /var subPath dance (only existed to move root-made changes into a non-root container). Main container runs as uid 0 with CAP_SYS_ADMIN,NET_ADMIN and allowPrivilegeEscalation=true; privileged stays false. Safe because the platform Kyverno policy sets hostUsers:false via the always-present docker-enabled marker. - Docker data PVC now backs /var/lib/docker (rootful), created unconditionally so it survives stop. - Drop the enable_docker parameter (docker is always on) and the DOCKER_HOST env (rootful uses the default socket; coder reaches it via the docker group). - script-agent-startup.sh reverts to a coder-user stub. Image still carries the (now-unused) rootless packages; cleanup deferred until this is confirmed working. Co-Authored-By: Claude --- .../kubernetes/homelab-workspace/configmap.tf | 6 +- .../homelab-workspace/deployment.tf | 166 ++++++------------ templates/kubernetes/homelab-workspace/env.tf | 10 +- .../homelab-workspace/parameters.tf | 11 -- .../homelab-workspace/script-agent-startup.sh | 49 +----- .../script-prepare-workspace.sh | 117 ------------ .../script-workspace-entrypoint.sh | 130 ++++++++++++++ 7 files changed, 195 insertions(+), 294 deletions(-) delete mode 100755 templates/kubernetes/homelab-workspace/script-prepare-workspace.sh create mode 100644 templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh diff --git a/templates/kubernetes/homelab-workspace/configmap.tf b/templates/kubernetes/homelab-workspace/configmap.tf index b17c0f49..2c07f2c4 100644 --- a/templates/kubernetes/homelab-workspace/configmap.tf +++ b/templates/kubernetes/homelab-workspace/configmap.tf @@ -8,8 +8,8 @@ resource "kubernetes_config_map_v1" "workspace_scripts" { } data = { - agent_startup_script = file("${path.cwd}/script-agent-startup.sh") - prepare_workspace_script = file("${path.cwd}/script-prepare-workspace.sh") - workspace_init_script = coder_agent.main.init_script + agent_startup_script = file("${path.cwd}/script-agent-startup.sh") + workspace_entrypoint_script = file("${path.cwd}/script-workspace-entrypoint.sh") + workspace_init_script = coder_agent.main.init_script } } diff --git a/templates/kubernetes/homelab-workspace/deployment.tf b/templates/kubernetes/homelab-workspace/deployment.tf index 6ed6eb5a..703be8c6 100644 --- a/templates/kubernetes/homelab-workspace/deployment.tf +++ b/templates/kubernetes/homelab-workspace/deployment.tf @@ -1,11 +1,3 @@ -locals { - # Marker label consumed by a Kyverno policy (separate repo) that injects - # `hostUsers: false` onto the pod — the hashicorp/kubernetes provider has no - # host_users attribute, so this label is the only way to request it. Absent - # (not "false") when docker is disabled so the policy simply doesn't match. - docker_labels = data.coder_parameter.enable_docker.value ? { "com.coder.workspace.docker-enabled" = "true" } : {} -} - resource "kubernetes_deployment_v1" "deployment" { count = data.coder_workspace.me.start_count @@ -30,7 +22,11 @@ resource "kubernetes_deployment_v1" "deployment" { template { metadata { - labels = merge(local.common_labels, local.pod_labels, local.docker_labels) + # The docker-enabled marker is what the platform Kyverno policy matches to + # inject `hostUsers: false` (the kubernetes provider has no host_users field). + # Docker is always enabled now, so the label is always present. hostUsers:false + # is what makes the root/CAP_SYS_ADMIN container below safe (namespaced root). + labels = merge(local.common_labels, local.pod_labels, { "com.coder.workspace.docker-enabled" = "true" }) } spec { dynamic "affinity" { @@ -51,14 +47,22 @@ resource "kubernetes_deployment_v1" "deployment" { } } automount_service_account_token = false - init_container { - name = "prepare-workspace" - command = ["/bin/bash", "/prepare-workspace-script.sh"] + container { + name = "workspace" + # Entrypoint runs as root: prepares the workspace (apt/homebrew), starts + # dockerd, then drops to the coder user to exec the agent. This replaces the + # old init-container + system-volume dance, which only existed to shuttle + # root-made changes into a non-root container -- unnecessary now. + command = ["/bin/bash", "/workspace-entrypoint.sh"] image = var.workspace_image # In test_mode the image is a mutable branch tag that gets rebuilt in place, # so a cached layer must not shadow a fresh push -> Always. Released versions # use immutable tags where IfNotPresent is correct (and avoids needless pulls). image_pull_policy = var.test_mode ? "Always" : "IfNotPresent" + env { + name = "CODER_AGENT_TOKEN" + value = coder_agent.main.token + } env { name = "SYSTEM_PACKAGES" value = length(local.validated_system_packages) > 0 ? join(" ", local.validated_system_packages) : "NONE" @@ -67,43 +71,6 @@ resource "kubernetes_deployment_v1" "deployment" { name = "HOMEBREW_PREFIX" value = local.homebrew_directory } - volume_mount { - mount_path = local.home_directory - name = "home" - sub_path = data.coder_workspace.me.name - } - volume_mount { - mount_path = local.homebrew_directory - name = "home" - sub_path = "${data.coder_workspace.me.name}/.linuxbrew" - } - volume_mount { - mount_path = "/prepare-workspace-script.sh" - name = "coder-scripts" - sub_path = "prepare_workspace_script" - } - volume_mount { - name = "system" - mount_path = "/updated" - } - security_context { - run_as_user = 0 - } - } - container { - name = "workspace" - command = ["/bin/bash", "/workspace-init.sh"] - image = var.workspace_image - # See init_container: Always in test_mode (mutable branch tag), else IfNotPresent. - image_pull_policy = var.test_mode ? "Always" : "IfNotPresent" - env { - name = "CODER_AGENT_TOKEN" - value = coder_agent.main.token - } - env { - name = "ENABLE_DOCKER" - value = tostring(data.coder_parameter.enable_docker.value) - } liveness_probe { exec { command = ["/bin/sh", "-c", "pgrep -f \"coder agent\" || exit 1"] @@ -123,12 +90,22 @@ resource "kubernetes_deployment_v1" "deployment" { } } security_context { - allow_privilege_escalation = false + # Root INSIDE the pod user namespace (hostUsers:false, injected by the + # platform Kyverno policy via the marker label above). Namespaced root maps + # to an unprivileged host uid, so these privileges are void on the host -- + # this is what lets a rootful dockerd run without being privileged-on-host. + # SYS_ADMIN/NET_ADMIN are what dockerd needs; escalation must be true for the + # caps to take effect. privileged stays false. The entrypoint drops to the + # coder user before handing off to the agent. + allow_privilege_escalation = true read_only_root_filesystem = false privileged = false - run_as_user = 10001 - run_as_group = 10001 - run_as_non_root = true + run_as_user = 0 + run_as_group = 0 + run_as_non_root = false + capabilities { + add = ["SYS_ADMIN", "NET_ADMIN"] + } } volume_mount { mount_path = local.home_directory @@ -150,38 +127,19 @@ resource "kubernetes_deployment_v1" "deployment" { sub_path = "agent_startup_script" } volume_mount { - mount_path = "/workspace-init.sh" + mount_path = "/workspace-entrypoint.sh" name = "coder-scripts" - sub_path = "workspace_init_script" - } - volume_mount { - mount_path = "/usr" - name = "system" - sub_path = "usr" + sub_path = "workspace_entrypoint_script" } volume_mount { - mount_path = "/etc" - name = "system" - sub_path = "etc" + mount_path = "/workspace-init.sh" + name = "coder-scripts" + sub_path = "workspace_init_script" } volume_mount { - mount_path = "/var" - name = "system" - sub_path = "var" - } - dynamic "volume_mount" { - for_each = data.coder_parameter.enable_docker.value ? toset(["docker-data"]) : [] - content { - name = "docker-data" - mount_path = "${local.home_directory}/.local/share/docker" - } - } - dynamic "volume_mount" { - for_each = data.coder_parameter.enable_docker.value ? toset(["xdg-runtime"]) : [] - content { - name = "xdg-runtime" - mount_path = "/run/user/10001" - } + # Rootful dockerd's data root; on the persistent docker-data PVC. + mount_path = "/var/lib/docker" + name = "docker-data" } } enable_service_links = false @@ -191,9 +149,11 @@ resource "kubernetes_deployment_v1" "deployment" { "kubernetes.io/arch" = "amd64" } security_context { - run_as_user = 10001 - run_as_group = 10001 - run_as_non_root = true + # Pod-level root, matching the container. Namespaced by hostUsers:false. + # fs_group stays 10001 so the coder user owns its home PVC contents. + run_as_user = 0 + run_as_group = 0 + run_as_non_root = false fs_group = 10001 fs_group_change_policy = "OnRootMismatch" } @@ -224,26 +184,10 @@ resource "kubernetes_deployment_v1" "deployment" { } } volume { - name = "system" - empty_dir { - size_limit = "10Gi" - } - } - dynamic "volume" { - for_each = data.coder_parameter.enable_docker.value ? toset(["docker-data"]) : [] - content { - name = "docker-data" - persistent_volume_claim { - claim_name = kubernetes_persistent_volume_claim_v1.docker_data[0].metadata[0].name - read_only = false - } - } - } - dynamic "volume" { - for_each = data.coder_parameter.enable_docker.value ? toset(["xdg-runtime"]) : [] - content { - name = "xdg-runtime" - empty_dir {} + name = "docker-data" + persistent_volume_claim { + claim_name = kubernetes_persistent_volume_claim_v1.docker_data[0].metadata[0].name + read_only = false } } } @@ -251,15 +195,17 @@ resource "kubernetes_deployment_v1" "deployment" { } } -# Persistent storage for the rootless Docker daemon's image/layer data. +# Persistent storage for the Docker daemon's image/layer data (/var/lib/docker). # -# Gated on enable_docker but deliberately NOT on start_count: like the external -# `coder-workspace-home` PVC, this must survive a workspace STOP (deployment -# scales to 0). Tying it to start_count would delete the volume — and every -# pulled image and built layer — on every stop. It is instead destroyed only on -# workspace DELETE or when docker is disabled. +# Deliberately NOT gated on start_count: like the external `coder-workspace-home` +# PVC, it must survive a workspace STOP (deployment scales to 0). Tying it to +# start_count would delete the volume -- and every pulled image and built layer -- +# on every stop. It is destroyed only on workspace DELETE. resource "kubernetes_persistent_volume_claim_v1" "docker_data" { - count = data.coder_parameter.enable_docker.value ? 1 : 0 + # count = 1 (not start_count): the deployment is gated on start_count and scales + # to 0 on stop, but this PVC must persist across stop so images/layers survive. + # terraform destroy (workspace delete) still removes it. + count = 1 wait_until_bound = false diff --git a/templates/kubernetes/homelab-workspace/env.tf b/templates/kubernetes/homelab-workspace/env.tf index 84b029ac..0fe783db 100644 --- a/templates/kubernetes/homelab-workspace/env.tf +++ b/templates/kubernetes/homelab-workspace/env.tf @@ -4,10 +4,6 @@ resource "coder_env" "welcome_message" { value = local.homebrew_directory } -resource "coder_env" "docker_host" { - count = data.coder_parameter.enable_docker.value ? 1 : 0 - - agent_id = coder_agent.main.id - name = "DOCKER_HOST" - value = "unix:///run/user/10001/docker.sock" -} +# Rootful dockerd listens on the default /var/run/docker.sock; the coder user +# reaches it via the `docker` group (see script-workspace-entrypoint.sh), so no +# DOCKER_HOST override is needed. diff --git a/templates/kubernetes/homelab-workspace/parameters.tf b/templates/kubernetes/homelab-workspace/parameters.tf index d531e419..c3215313 100644 --- a/templates/kubernetes/homelab-workspace/parameters.tf +++ b/templates/kubernetes/homelab-workspace/parameters.tf @@ -61,17 +61,6 @@ data "coder_parameter" "system_packages" { type = "list(string)" } -data "coder_parameter" "enable_docker" { - name = "enable_docker" - - default = false - display_name = "Enable Docker" - description = "Run a rootless Docker daemon inside the workspace" - icon = "/icon/docker.svg" - mutable = true - type = "bool" -} - locals { validated_system_packages = (data.coder_parameter.system_packages.value != "") ? [ diff --git a/templates/kubernetes/homelab-workspace/script-agent-startup.sh b/templates/kubernetes/homelab-workspace/script-agent-startup.sh index 4dee0408..bdf3e7a2 100644 --- a/templates/kubernetes/homelab-workspace/script-agent-startup.sh +++ b/templates/kubernetes/homelab-workspace/script-agent-startup.sh @@ -1,47 +1,9 @@ #!/bin/bash +# Runs as the `coder` user via the Coder agent's startup_script hook, after the +# agent is up. This is the place for per-user setup that must run unprivileged. +# (Docker + system prep now happen earlier, as root, in script-workspace-entrypoint.sh.) set -eo pipefail - -setup_docker() { - export XDG_RUNTIME_DIR="/run/user/10001" - export DOCKER_HOST="unix://${XDG_RUNTIME_DIR}/docker.sock" - # dockerd-rootless.sh and dockerd-rootless-setuptool.sh ship in /usr/bin - export PATH="/usr/bin:${PATH}" - - if ! command -v dockerd-rootless.sh > /dev/null 2>&1; then - echo 'dockerd-rootless.sh not found (docker-ce-rootless-extras missing from image); skipping docker setup.' - return 1 - fi - - echo 'Ensuring XDG_RUNTIME_DIR exists...' - mkdir -p "${XDG_RUNTIME_DIR}" - # The dir is an emptyDir mount owned by root but group-writable via fsGroup=10001, - # so it is already usable by coder; chmod would fail (coder is not the owner) and - # that is harmless -- keep it non-fatal rather than emitting a scary error. - chmod 700 "${XDG_RUNTIME_DIR}" 2> /dev/null || true - - echo 'Installing rootless docker (idempotent; tolerate re-run)...' - dockerd-rootless-setuptool.sh install 2>&1 | sed -E -n 's|^| |p' || true - - echo 'Starting rootless dockerd...' - # Storage driver is auto-detected by dockerd with preference order - # overlay2 -> fuse-overlayfs -> vfs. When overlay2 is unavailable in this - # (userns) environment docker falls back on its own; pass `--storage-driver` - # to dockerd-rootless.sh only if a specific driver must be forced. - setsid dockerd-rootless.sh > "${HOME}/.local/share/docker/dockerd.log" 2>&1 & - - echo 'Waiting up to ~30s for docker to become ready...' - for _ in $(seq 1 30); do - if docker info > /dev/null 2>&1; then - echo 'Docker is ready.' - return 0 - fi - sleep 1 - done - echo "Docker did not become ready; see ${HOME}/.local/share/docker/dockerd.log" - return 1 -} - main() { if [[ ! -s ~/.bashrc ]]; then echo "Setting up starter bash rc scripts from /etc/skel..." @@ -50,11 +12,6 @@ main() { echo 'set -o allexport; source /etc/environment; set +o allexport' >> ~/.bashrc echo '------------------------------------------------------------' fi - if [[ "${ENABLE_DOCKER:-false}" == "true" ]]; then - echo "Setting up rootless Docker..." - # Non-fatal: a docker failure must not abort agent startup. - setup_docker 2>&1 | sed -E -n 's|^| |p' || true - fi echo 'Done' } diff --git a/templates/kubernetes/homelab-workspace/script-prepare-workspace.sh b/templates/kubernetes/homelab-workspace/script-prepare-workspace.sh deleted file mode 100755 index 3140d1ee..00000000 --- a/templates/kubernetes/homelab-workspace/script-prepare-workspace.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -set -euo pipefail - -install_homebrew() { - local brew_user="$1" - export HOMEBREW_CELLAR="${HOMEBREW_PREFIX}/Cellar" - export HOMEBREW_NO_ANALYTICS=1 - export PATH="${HOMEBREW_PREFIX}/bin:${HOMEBREW_PREFIX}/sbin:$PATH" - # init container runs as root but homebrew doesn't support running as root - # we don't want to give any users within the coder workspace the ability to sudo, so this is a workaround - # see: https://github.com/Homebrew/install/blob/7e3a5202cd6d783a2464e387433c4c72acdb0f49/install.sh#L366 - touch /.dockerenv - HOME="/home/${brew_user}" NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -} - -cleanup_homebrew() { - echo "Cleaning up..." - echo " Cache..." - find "${HOMEBREW_CACHE}/" -mindepth 1 -maxdepth 1 -exec rm -rf {} \; || true - echo " Prefix..." - find "${HOMEBREW_PREFIX}/" -mindepth 1 -maxdepth 1 -exec rm -rf {} \; || true -} - -prepare_homebrew() { - echo "Creating directories and setting permissions..." - if [[ ! -d "${HOMEBREW_CACHE}" ]]; then - echo "Creating cache directory..." - mkdir -p "${HOMEBREW_CACHE}" - fi - chown -R ${brew_user}:root "${HOMEBREW_CACHE}" - chown -R ${brew_user}:root $(dirname "${HOMEBREW_PREFIX}") -} - -ensure_permissions() { - local brew_user="$1" - echo "Making brew installation available to both ${brew_user} user and root..." - chown -R ${brew_user}:root $(dirname "${HOMEBREW_PREFIX}") - chown -R ${brew_user}:root "${HOMEBREW_CACHE}" - echo "Ensuring ${brew_user} user retains ownership of top level directories within their home..." - chown ${brew_user}:${brew_user} /home/${brew_user} - chown ${brew_user}:${brew_user} /home/${brew_user}/.cache -} - -setup_homebrew() { - local brew_user="coder" - export HOMEBREW_CACHE="/home/${brew_user}/.cache/Homebrew" - - echo '------------------------------------------------------------' - echo 'Checking for brew installation...' - local brew_file_count=$(find "${HOMEBREW_PREFIX}" -maxdepth 3 -type f | wc -l) - if [[ "${brew_file_count}" -gt "0" && -f "${HOMEBREW_PREFIX}/bin/brew" ]]; then - echo 'Brew installation already exists... skipping.' - else - echo 'No existing brew installation, proceeding w/ installation...' - echo '------------------------------------------------------------' - echo 'Preparing for homebrew...' - cleanup_homebrew 2>&1 | sed -E -n 's|^| |p' - prepare_homebrew 2>&1 | sed -E -n 's|^| |p' - echo '------------------------------------------------------------' - echo 'Starting homebrew installation...' - install_homebrew "${brew_user}" 2>&1 | sed -E -n 's|^| |p' - fi - echo '------------------------------------------------------------' - echo "Ensuring directory permissions..." - ensure_permissions "${brew_user}" 2>&1 | sed -E -n 's|^| |p' - echo '------------------------------------------------------------' - echo 'Done' -} - -setup_system_packages() { - echo '------------------------------------------------------------' - echo 'Running apt-get update...' - apt-get update 2>&1 | sed -E -n 's|^| |p' - echo 'Running apt-file update...' - apt-file update 2>&1 | sed -E -n 's|^| |p' - echo '------------------------------------------------------------' - echo 'Installing additinal apt packages...' - echo " Packages: $SYSTEM_PACKAGES" - echo - if [[ "$SYSTEM_PACKAGES" != "NONE" ]]; then - DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -yq $SYSTEM_PACKAGES 2>&1 | sed -E -n 's|^| |p' - fi - echo '------------------------------------------------------------' - echo 'Rsyncing (etc, usr, var) to /updated...' - rsync -aH --stats /etc /usr /var /updated 2>&1 | sed -E -n 's|^| |p' - echo '------------------------------------------------------------' - echo 'Updated system size...' - du -h -d 1 /updated 2>&1 | sed -E -n 's|^| |p' - echo '------------------------------------------------------------' - echo 'Done' -} - -prepare_environment() { - echo '------------------------------------------------------------' - grep -v '^PATH=' /etc/environment > /tmp/environment.bak - local existing_system_path="$(grep '^PATH=' /etc/environment | cut -d'=' -f2)" - local updated_system_path="${HOMEBREW_PREFIX}/bin:${HOMEBREW_PREFIX}/sbin:${existing_system_path}" - echo PATH=${updated_system_path} >> /tmp/environment.bak - sort /tmp/environment.bak > /updated/etc/environment - rm /tmp/environment.bak - cat /updated/etc/environment - echo '------------------------------------------------------------' - echo 'Done' -} - -main() { - echo "Setting up system packages..." - setup_system_packages | sed -E -n 's|^| |p' - echo - echo "Setting up homebrew..." - setup_homebrew | sed -E -n 's|^| |p' - echo - echo "Setting up workspace environment variables..." - prepare_environment | sed -E -n 's|^| |p' -} - -main diff --git a/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh b/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh new file mode 100644 index 00000000..6671efc0 --- /dev/null +++ b/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# Main container entrypoint. Runs as root (uid 0) INSIDE the pod's user +# namespace (hostUsers=false), so "root" here maps to an unprivileged uid on the +# host -- see DESIGN-DIND.md. Responsibilities, in order: +# 1. prepare the workspace (extra apt packages, Homebrew, PATH) -- previously an +# init container; now done here directly because this container is already root. +# 2. start a rootful dockerd (non-fatal: the workspace must come up even if it fails). +# 3. drop to the unprivileged `coder` user and exec the Coder agent, so everything +# the user interacts with runs as `coder` exactly as before. +set -eo pipefail + +CODER_USER="coder" +CODER_UID="10001" +CODER_GID="10001" +HOMEBREW_PREFIX="${HOMEBREW_PREFIX:-/home/linuxbrew/.linuxbrew}" + +# -------------------------------------------------------------------------------- +install_homebrew() { + export HOMEBREW_CELLAR="${HOMEBREW_PREFIX}/Cellar" + export HOMEBREW_NO_ANALYTICS=1 + export PATH="${HOMEBREW_PREFIX}/bin:${HOMEBREW_PREFIX}/sbin:$PATH" + # Homebrew refuses to install as root; run its installer as the coder user. + # (/.dockerenv makes the installer treat this as a container and skip sudo prompts.) + touch /.dockerenv + setpriv --reuid "${CODER_UID}" --regid "${CODER_GID}" --init-groups \ + env HOME="/home/${CODER_USER}" NONINTERACTIVE=1 \ + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +} + +setup_homebrew() { + export HOMEBREW_CACHE="/home/${CODER_USER}/.cache/Homebrew" + echo 'Checking for brew installation...' + local brew_file_count + brew_file_count=$(find "${HOMEBREW_PREFIX}" -maxdepth 3 -type f 2> /dev/null | wc -l) + if [[ "${brew_file_count}" -gt "0" && -f "${HOMEBREW_PREFIX}/bin/brew" ]]; then + echo 'Brew installation already exists... skipping.' + else + echo 'No existing brew installation, proceeding w/ installation...' + mkdir -p "${HOMEBREW_CACHE}" + install_homebrew 2>&1 | sed -E -n 's|^| |p' + fi + echo 'Ensuring ownership of brew + home top-level dirs by coder...' + chown -R "${CODER_USER}:root" "$(dirname "${HOMEBREW_PREFIX}")" "${HOMEBREW_CACHE}" + chown "${CODER_USER}:${CODER_USER}" "/home/${CODER_USER}" "/home/${CODER_USER}/.cache" +} + +setup_system_packages() { + echo "Installing additional apt packages: ${SYSTEM_PACKAGES:-NONE}" + if [[ "${SYSTEM_PACKAGES:-NONE}" != "NONE" ]]; then + apt-get update 2>&1 | sed -E -n 's|^| |p' + apt-file update 2>&1 | sed -E -n 's|^| |p' || true + # shellcheck disable=SC2086 + DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -yq ${SYSTEM_PACKAGES} 2>&1 | sed -E -n 's|^| |p' + fi +} + +prepare_environment() { + # Extend the system PATH (in /etc/environment) with Homebrew, in place -- no + # /updated staging volume any more since this container owns its own rootfs. + local existing updated + existing="$(grep '^PATH=' /etc/environment | cut -d'=' -f2)" + updated="${HOMEBREW_PREFIX}/bin:${HOMEBREW_PREFIX}/sbin:${existing}" + grep -v '^PATH=' /etc/environment > /tmp/environment.new + echo "PATH=${updated}" >> /tmp/environment.new + sort /tmp/environment.new > /etc/environment + rm -f /tmp/environment.new +} + +# -------------------------------------------------------------------------------- +setup_docker() { + if ! command -v dockerd > /dev/null 2>&1; then + echo 'dockerd not found in image; skipping docker setup.' + return 1 + fi + # Let the coder user reach the daemon socket. + groupadd --force docker + usermod --append --groups docker "${CODER_USER}" + + echo 'Starting dockerd...' + # Rootful dockerd. CAP_SYS_ADMIN/NET_ADMIN (granted on the container, valid only + # inside the userns) cover what it needs. Data lives on the docker-data PVC at + # /var/lib/docker. overlay2 is the default/expected driver. + mkdir -p /var/lib/docker + setsid dockerd > /var/log/dockerd.log 2>&1 & + + echo 'Waiting up to ~30s for docker to become ready...' + local waited=0 + while [[ "${waited}" -lt 30 ]]; do + waited=$((waited + 1)) + if docker info > /dev/null 2>&1; then + echo 'Docker is ready.' + # socket is created 0660 root:docker; ensure the group can use it + chgrp docker /var/run/docker.sock 2> /dev/null || true + chmod 660 /var/run/docker.sock 2> /dev/null || true + return 0 + fi + sleep 1 + done + echo 'Docker did not become ready; see /var/log/dockerd.log' + return 1 +} + +# -------------------------------------------------------------------------------- +main() { + echo '=== workspace entrypoint (root) ===' + + echo '--- system packages ---' + setup_system_packages | sed -E -n 's|^| |p' || echo ' (system package setup failed; continuing)' + + echo '--- homebrew ---' + setup_homebrew 2>&1 | sed -E -n 's|^| |p' || echo ' (homebrew setup failed; continuing)' + + echo '--- environment ---' + prepare_environment || echo ' (environment prep failed; continuing)' + + echo '--- docker ---' + # Non-fatal by design: a docker failure must never stop the workspace coming up. + setup_docker 2>&1 | sed -E -n 's|^| |p' || echo ' (docker setup failed; workspace continues without docker)' + + echo '--- starting coder agent as coder ---' + # Drop from root to the coder user and hand off to the agent. exec so the agent + # becomes the container's main process (as it was when the container ran as coder). + # --init-groups picks up the docker group added above. Preserve the agent env. + exec setpriv --reuid "${CODER_UID}" --regid "${CODER_GID}" --init-groups \ + env HOME="/home/${CODER_USER}" USER="${CODER_USER}" \ + CODER_AGENT_TOKEN="${CODER_AGENT_TOKEN}" \ + /bin/bash /workspace-init.sh +} + +main From ba4c675d67a383856c3aa98e3f23c81c61550390 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Tue, 21 Jul 2026 00:18:17 +0000 Subject: [PATCH 7/9] fix: bind-mount docker data root instead of using a PVC mount dockerd failed to extract image layers: 'mount ... bind ... permission denied', with 'could not setup daemon root propagation to shared' earlier. Root cause: the /var/lib/docker PVC is mounted by the kubelet OUTSIDE the pod user namespace, so it is a locked private mount -- dockerd cannot make it shared or bind-mount snapshots within it, even as namespaced root with CAP_SYS_ADMIN (verified: findmnt shows PROPAGATION=private, mount --make-rshared -> EPERM). Fix: drop the /var/lib/docker PVC entirely. The entrypoint instead bind-mounts a dir under the home PVC (/home/coder/.var/docker) onto /var/lib/docker inside this container's own mount namespace -- an unlocked mount dockerd CAN manage -- and makes it shared. Data still persists across stop via the home PVC. Both dirs are set root:root 0711 (dockerd rejects a group/world-accessible data root; the home PVC is otherwise fsGroup-owned by the coder group). Co-Authored-By: Claude --- .../homelab-workspace/deployment.tf | 48 ++----------------- .../script-workspace-entrypoint.sh | 24 ++++++++-- 2 files changed, 26 insertions(+), 46 deletions(-) diff --git a/templates/kubernetes/homelab-workspace/deployment.tf b/templates/kubernetes/homelab-workspace/deployment.tf index 703be8c6..aa4bd2c3 100644 --- a/templates/kubernetes/homelab-workspace/deployment.tf +++ b/templates/kubernetes/homelab-workspace/deployment.tf @@ -136,11 +136,11 @@ resource "kubernetes_deployment_v1" "deployment" { name = "coder-scripts" sub_path = "workspace_init_script" } - volume_mount { - # Rootful dockerd's data root; on the persistent docker-data PVC. - mount_path = "/var/lib/docker" - name = "docker-data" - } + # No /var/lib/docker volume: a kubelet-mounted PVC there is a locked, + # private mount that dockerd cannot make shared or bind within (EPERM in + # the userns). Instead the entrypoint bind-mounts a dir under the home PVC + # to /var/lib/docker inside this container's own mount namespace, which + # dockerd can manage -- and it still persists across stop via the home PVC. } enable_service_links = false hostname = lower(replace(data.coder_workspace.me.name, "/[^a-zA-Z0-9]/", "-")) @@ -183,44 +183,6 @@ resource "kubernetes_deployment_v1" "deployment" { default_mode = "0750" } } - volume { - name = "docker-data" - persistent_volume_claim { - claim_name = kubernetes_persistent_volume_claim_v1.docker_data[0].metadata[0].name - read_only = false - } - } - } - } - } -} - -# Persistent storage for the Docker daemon's image/layer data (/var/lib/docker). -# -# Deliberately NOT gated on start_count: like the external `coder-workspace-home` -# PVC, it must survive a workspace STOP (deployment scales to 0). Tying it to -# start_count would delete the volume -- and every pulled image and built layer -- -# on every stop. It is destroyed only on workspace DELETE. -resource "kubernetes_persistent_volume_claim_v1" "docker_data" { - # count = 1 (not start_count): the deployment is gated on start_count and scales - # to 0 on stop, but this PVC must persist across stop so images/layers survive. - # terraform destroy (workspace delete) still removes it. - count = 1 - - wait_until_bound = false - - metadata { - name = "coder-${data.coder_workspace.me.id}-docker" - namespace = "coder" - labels = merge(local.common_labels, local.pod_labels) - } - - spec { - access_modes = ["ReadWriteOnce"] - storage_class_name = "sc-longhorn-local-non-replicated" - resources { - requests = { - storage = "40Gi" } } } diff --git a/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh b/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh index 6671efc0..38b92690 100644 --- a/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh +++ b/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh @@ -76,11 +76,29 @@ setup_docker() { groupadd --force docker usermod --append --groups docker "${CODER_USER}" + # Docker's data root must live on a mount that dockerd can make shared and + # bind-mount within (it does this for every layer/snapshot). A PVC mounted by + # the kubelet is created OUTSIDE this pod's user namespace, so it is a locked, + # private mount: `mount --make-shared` and the snapshot bind-mounts both fail + # with EPERM even as (namespaced) root. A bind mount we create ourselves inside + # this container's mount namespace is not locked, so dockerd can manage it. + # + # Back the data root with a dir under the home PVC (persists across stop) and + # bind it to /var/lib/docker. Both dirs must be root:root 0711 -- dockerd + # rejects a world/group-accessible data root, and the home PVC is otherwise + # fsGroup-owned by the coder group. + local docker_data="/home/${CODER_USER}/.var/docker" + mkdir -p "${docker_data}" /var/lib/docker + chown root:root "${docker_data}" /var/lib/docker + chmod 0711 "${docker_data}" /var/lib/docker + mount --bind "${docker_data}" /var/lib/docker + # dockerd also sets this itself; do it up front so the data root is shared before + # it starts. Non-fatal -- if it fails, dockerd's own attempt is what matters. + mount --make-shared /var/lib/docker || true + echo 'Starting dockerd...' # Rootful dockerd. CAP_SYS_ADMIN/NET_ADMIN (granted on the container, valid only - # inside the userns) cover what it needs. Data lives on the docker-data PVC at - # /var/lib/docker. overlay2 is the default/expected driver. - mkdir -p /var/lib/docker + # inside the userns) cover what it needs. overlay2 is the default/expected driver. setsid dockerd > /var/log/dockerd.log 2>&1 & echo 'Waiting up to ~30s for docker to become ready...' From fd6ad516b041f73a8438cd0bf0b71c3d7ac37caa Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Tue, 21 Jul 2026 00:51:24 +0000 Subject: [PATCH 8/9] fix: back /var/lib/docker with an emptyDir instead of a bind mount The entrypoint's in-container bind mount was silently blocked by the container AppArmor profile (mount is denied there), so it never took effect. Mount the docker data root as a kubelet-provided emptyDir instead: it is a plain non-overlay fs set up outside the container, avoiding both overlay-on-overlay (the container rootfs is overlayfs) and the in-container mount restriction. Entrypoint just sets ownership (root:root 0711); no in-container mount needed. Tradeoff: emptyDir does not persist across workspace stop -- revisit persistence once the runtime is confirmed working end to end. Co-Authored-By: Claude --- .../homelab-workspace/deployment.tf | 24 ++++++++++++++---- .../script-workspace-entrypoint.sh | 25 +++++-------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/templates/kubernetes/homelab-workspace/deployment.tf b/templates/kubernetes/homelab-workspace/deployment.tf index aa4bd2c3..591a4b66 100644 --- a/templates/kubernetes/homelab-workspace/deployment.tf +++ b/templates/kubernetes/homelab-workspace/deployment.tf @@ -136,11 +136,15 @@ resource "kubernetes_deployment_v1" "deployment" { name = "coder-scripts" sub_path = "workspace_init_script" } - # No /var/lib/docker volume: a kubelet-mounted PVC there is a locked, - # private mount that dockerd cannot make shared or bind within (EPERM in - # the userns). Instead the entrypoint bind-mounts a dir under the home PVC - # to /var/lib/docker inside this container's own mount namespace, which - # dockerd can manage -- and it still persists across stop via the home PVC. + volume_mount { + # Docker data root on a kubelet-mounted emptyDir: a plain (non-overlay) + # filesystem, so containerd's overlayfs snapshotter isn't stacking overlay + # on overlay (which the container rootfs would otherwise force). The kubelet + # sets this mount up outside the container, so it does not hit the in-container + # AppArmor mount restrictions. NOTE: emptyDir does not persist across stop. + mount_path = "/var/lib/docker" + name = "docker-data" + } } enable_service_links = false hostname = lower(replace(data.coder_workspace.me.name, "/[^a-zA-Z0-9]/", "-")) @@ -183,6 +187,16 @@ resource "kubernetes_deployment_v1" "deployment" { default_mode = "0750" } } + volume { + # Docker data root. emptyDir (not PVC) so it is a plain fs the kubelet + # mounts outside the container -- avoids overlay-on-overlay and the locked + # -mount / in-container AppArmor problems. Persistence across stop is a + # follow-up once the runtime is confirmed working. + name = "docker-data" + empty_dir { + size_limit = "40Gi" + } + } } } } diff --git a/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh b/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh index 38b92690..ba9fe080 100644 --- a/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh +++ b/templates/kubernetes/homelab-workspace/script-workspace-entrypoint.sh @@ -76,25 +76,12 @@ setup_docker() { groupadd --force docker usermod --append --groups docker "${CODER_USER}" - # Docker's data root must live on a mount that dockerd can make shared and - # bind-mount within (it does this for every layer/snapshot). A PVC mounted by - # the kubelet is created OUTSIDE this pod's user namespace, so it is a locked, - # private mount: `mount --make-shared` and the snapshot bind-mounts both fail - # with EPERM even as (namespaced) root. A bind mount we create ourselves inside - # this container's mount namespace is not locked, so dockerd can manage it. - # - # Back the data root with a dir under the home PVC (persists across stop) and - # bind it to /var/lib/docker. Both dirs must be root:root 0711 -- dockerd - # rejects a world/group-accessible data root, and the home PVC is otherwise - # fsGroup-owned by the coder group. - local docker_data="/home/${CODER_USER}/.var/docker" - mkdir -p "${docker_data}" /var/lib/docker - chown root:root "${docker_data}" /var/lib/docker - chmod 0711 "${docker_data}" /var/lib/docker - mount --bind "${docker_data}" /var/lib/docker - # dockerd also sets this itself; do it up front so the data root is shared before - # it starts. Non-fatal -- if it fails, dockerd's own attempt is what matters. - mount --make-shared /var/lib/docker || true + # /var/lib/docker is a kubelet-mounted emptyDir (see deployment.tf): a plain, + # non-overlay filesystem, so containerd's overlayfs snapshotter is not trying to + # stack overlay-on-overlay (which is what the container rootfs would force). The + # dir must be root:root 0711 -- dockerd rejects a group/world-accessible data root. + chown root:root /var/lib/docker + chmod 0711 /var/lib/docker echo 'Starting dockerd...' # Rootful dockerd. CAP_SYS_ADMIN/NET_ADMIN (granted on the container, valid only From cd9250a7ae48637b3ae8565f662867bf725c8070 Mon Sep 17 00:00:00 2001 From: Peter Pathirana Date: Tue, 21 Jul 2026 01:05:55 +0000 Subject: [PATCH 9/9] test: set privileged=true to confirm AppArmor is blocking dockerd mounts emptyDir at /var/lib/docker gives containerd an ext4 (non-overlay) data root, yet image extraction still fails 'permission denied' on containerd's internal bind mount -- proving it is NOT overlay-on-overlay but the cri-containerd AppArmor profile's deny-mount rule. privileged=true sets the profile to unconfined; if docker then works, the durable fix is appArmorProfile:Unconfined via Kyverno (provider has no field for it), without going fully privileged. Testing step, not the final posture. Co-Authored-By: Claude --- .../homelab-workspace/deployment.tf | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/templates/kubernetes/homelab-workspace/deployment.tf b/templates/kubernetes/homelab-workspace/deployment.tf index 591a4b66..7e5260b2 100644 --- a/templates/kubernetes/homelab-workspace/deployment.tf +++ b/templates/kubernetes/homelab-workspace/deployment.tf @@ -92,20 +92,21 @@ resource "kubernetes_deployment_v1" "deployment" { security_context { # Root INSIDE the pod user namespace (hostUsers:false, injected by the # platform Kyverno policy via the marker label above). Namespaced root maps - # to an unprivileged host uid, so these privileges are void on the host -- - # this is what lets a rootful dockerd run without being privileged-on-host. - # SYS_ADMIN/NET_ADMIN are what dockerd needs; escalation must be true for the - # caps to take effect. privileged stays false. The entrypoint drops to the - # coder user before handing off to the agent. + # to an unprivileged host uid, so even privileged is void on the host. + # + # privileged=true (TEST): the cri-containerd AppArmor profile default-denies + # the mount() syscall, which blocks containerd's overlayfs snapshotter from + # bind/overlay-mounting image layers (confirmed: fails on an ext4 emptyDir data + # root, so it is NOT overlay-on-overlay -- it is AppArmor). privileged sets the + # profile to unconfined (and grants all caps), which should unblock it. If this + # works, the durable fix is appArmorProfile:Unconfined via the Kyverno policy + # (the provider has no app_armor_profile field), keeping caps narrow. allow_privilege_escalation = true read_only_root_filesystem = false - privileged = false + privileged = true run_as_user = 0 run_as_group = 0 run_as_non_root = false - capabilities { - add = ["SYS_ADMIN", "NET_ADMIN"] - } } volume_mount { mount_path = local.home_directory