test(features): run feature tests against environment services, starting with Keycloak

.github/workflows/branch-checks.yml

@@ -234,6 +234,8 @@ jobs:
           cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check
+          cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh -- --check
+          cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check
 
   cargo-lockfiles:
     name: Cargo lockfiles
@@ -296,6 +298,8 @@ jobs:
           cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings
 
   rust-test:
     name: Rust test (${{ matrix.system }})

.github/workflows/e2e-suites.yml

@@ -50,6 +50,11 @@ jobs:
             family: conformance
             package: sandbox
             runner: ubuntu-24.04
+          - environment: ubuntu-docker-rootful-keycloak
+            installer: deb
+            family: features
+            package: keycloak-provider-refresh
+            runner: ubuntu-24.04
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 90
     steps:
@@ -77,13 +82,17 @@ jobs:
         uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
         with:
           path: ~/.cache/tmachine
-          # tmachine keys its disk layers on a hash of the Ansible tree and
-          # the cache versions in setup.rs and install.rs. actions/cache
-          # doesn't save after an exact key hit, so a key that misses one of
-          # those would rebuild the changed layers on every run. No
-          # restore-keys fallback: restoring an older entry would carry its
-          # stale disks into the new one, which then only grows.
-          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs') }}
+          # tmachine keys its disk layers on a hash of the Ansible tree, the
+          # environments' playbook lists and setup inputs (config.nix, and
+          # files such as the Keycloak realm file) and the cache versions in
+          # setup.rs and install.rs. actions/cache doesn't save after an
+          # exact key hit, so a key that misses one of those would rebuild the
+          # changed layers on every run. Environments share no layers, and
+          # only the first job to save a key gets its disks in, so each
+          # environment and installer needs its own key. No restore-keys
+          # fallback: restoring an older entry would carry its stale disks
+          # into the new one, which then only grows.
+          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs', 'scripts/keycloak-realm.json') }}
 
       - name: Download integration inputs
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1

TESTING.md

@@ -230,6 +230,63 @@ an explicit matrix and runs the runner once per row. Branch E2E calls it when
 the `test:e2e` label is set, and a failure fails the Core E2E result. To run
 a suite in another environment on CI, add a row.
 
+### Adding a feature suite
+
+Feature suites live in the `tests/suites/features` workspace, one crate per
+suite, with a name that says what the suite tests.
+`tests/suites/features/keycloak-provider-refresh` is the example to copy.
+
+1. Add a crate under `tests/suites/features/<name>` that depends on
+   `openshell-testkit`, and add it to `members` in
+   `tests/suites/features/Cargo.toml`.
+2. Write the tests in `tests/*.rs` as `#[testkit::test]` functions.
+3. Re-lock the workspace: `cargo metadata --manifest-path
+   tests/suites/features/Cargo.toml`, then
+   `uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix`.
+4. Add `cargo fmt` and `cargo clippy` lines for the package (`-p <name>`) to
+   `tasks/rust.toml` and to the Format and Lint steps in
+   `.github/workflows/branch-checks.yml`.
+5. Add a row for the suite to the matrix in `.github/workflows/e2e-suites.yml`.
+
+If the feature only needs the gateway, run the suite in an existing
+environment such as `ubuntu-docker-rootful`, and skip the rest of this
+section.
+
+**Suites that need a service.** A feature that needs something besides the
+gateway, such as an OIDC issuer, runs in a tmachine environment that provides
+it. The environment and the suite share a contract: a set of
+`OPENSHELL_TEST_<SERVICE>_*` environment variables that say how to reach the
+service.
+
+- **Environment.** Add an environment to `tests/tmachine/config.nix` whose
+  setup playbook starts the service (with a restart policy, since setup is
+  cached as a disk) and writes the variables as `KEY=VALUE` lines to
+  `/etc/openshell-test/env.d/<service>.env`. Pass files the playbook needs
+  through the environment's setup `inputs`, and add them to the tmachine disk
+  cache key in `e2e-suites.yml`. The runner rejects names without the
+  `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, so
+  values never appear in a runner command. Values are taken literally, without
+  quoting.
+- **Fixture.** Put the service's fixture in a support crate under
+  `tests/support/<service>`, so any suite can depend on it, or reuse the
+  existing one. It defines the variable names as constants and a
+  `<Service>Context::new(cx)` that reads them, waits for the service, and
+  wraps the testkit `Context`. Each test calls it on its first line.
+  Nextest runs each test in its own process, so the fixture builds its
+  clients there from the variables. A missing variable fails the test; it
+  never skips it.
+- **Documentation.** Document the variables in the fixture's module docs, so
+  anyone can run the suite against their own deployment by setting them.
+
+The Keycloak fixture is `tests/support/keycloak`. Run the suite that uses it
+with:
+
+```shell
+nix run .#openshell-test-runner -- \
+  --environment ubuntu-docker-rootful-keycloak --installer deb \
+  --family features --package keycloak-provider-refresh
+```
+
 ### Python E2E (`e2e/python/`)
 
 `mise run e2e:python` builds `openshell/e2e-python:dev` from

tasks/rust.toml

@@ -40,6 +40,8 @@ run = [
   "cargo clippy --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings",
   "cargo clippy --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings",
   "cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings",
+  "cargo clippy --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh --all-targets -- -D warnings",
+  "cargo clippy --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings",
 ]
 run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 lint native"
 hide = true
@@ -56,6 +58,8 @@ run = [
   "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all",
   "cargo fmt --manifest-path tests/runner/Cargo.toml --all",
   "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox",
+  "cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh",
+  "cargo fmt --manifest-path tests/support/Cargo.toml --all",
 ]
 hide = true
 
@@ -71,6 +75,8 @@ run = [
   "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check",
   "cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check",
   "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check",
+  "cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh -- --check",
+  "cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check",
 ]
 hide = true
 

tasks/scripts/check_cargo_lockfile_alignment.py

@@ -35,8 +35,10 @@
 # Workspaces whose shared crates must match the root lockfile. Add a
 # workspace here once its lockfile has been aligned with `--fix`.
 ALIGNED_WORKSPACES = (
+    "tests/support",
     "tests/testkit",
     "tests/suites/conformance",
+    "tests/suites/features",
 )
 
 

tests/runner/src/execute.rs

@@ -7,7 +7,9 @@
 //!
 //! Tests find the gateway the way the CLI does, so the runner points them at
 //! the one the installer registered with `OPENSHELL_GATEWAY` and at the
-//! installed CLI with `OPENSHELL_BIN`.
+//! installed CLI with `OPENSHELL_BIN`. Anything else the environment provides,
+//! such as Keycloak, the tests learn from the variables it publishes (see
+//! [`crate::test_env`]).
 
 use std::fmt;
 use std::path::Path;
@@ -19,6 +21,7 @@ use tmachine::Session;
 use tmachine::config::Config;
 
 use crate::bundle::{Bundle, PROFILE, READY_BINARY};
+use crate::test_env;
 
 const GUEST_DIR: &str = "/var/tmp/openshell-tests";
 /// The nextest log's name, in the guest directory and in the results.
@@ -225,6 +228,9 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
         format!("OPENSHELL_GATEWAY={}", run.gateway),
         format!("OPENSHELL_BIN={openshell_bin}"),
     ];
+    test_env::check(session)
+        .await
+        .context("check the variables the environment publishes for tests")?;
 
     let guest = GUEST_DIR;
     checked(
@@ -289,9 +295,15 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
         nextest.extend(["-E", filter]);
     }
     // nextest writes its output to a file in the guest, so a run that times
-    // out or loses SSH still leaves a log for `collect_results`.
+    // out or loses SSH still leaves a log for `collect_results`. The guest
+    // shell exports the published variables itself, so their values, which
+    // can be credentials, never appear in the command (or its errors).
     let log = format!("{guest}/{NEXTEST_LOG}");
-    let mut command = vec!["sh", "-c", r#"exec "$@" >"$0" 2>&1"#, &log];
+    let script = format!(
+        r#"exec >"$0" 2>&1; {}; exec "$@""#,
+        test_env::export_script(test_env::ENV_DIR)
+    );
+    let mut command = vec!["sh", "-c", &script, &log];
     command.extend(nextest);
     let output = session.exec_timeout(&command, RUN_TIMEOUT).await?;
     Ok(Status::of_nextest(

tests/runner/src/main.rs

@@ -17,6 +17,7 @@ use crate::execute::{Run, Status};
 
 mod bundle;
 mod execute;
+mod test_env;
 
 /// Build a test family for the guest, boot a tmachine environment with
 /// OpenShell installed, run the tests in it, and check the results.