diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bdafcd13a..ac122ce18 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -395,6 +395,10 @@ jobs: env: DATA_DIR: /tmp/hypeman-ci-${{ github.run_id }} steps: + - name: Report macOS host version + run: | + sw_vers + sysctl -n machdep.cpu.brand_string - uses: actions/checkout@v4 with: repository: ${{ env.TEST_SOURCE_REPO }} diff --git a/README.md b/README.md index 70d3ee66f..88dd98ec6 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,19 @@ hypeman restore my-app hypeman rm --force --all ``` +### Experimental live forks on macOS + +```bash +hypeman fork --from-running --target-state Running source copy +``` + +VZ live forks preserve running Linux processes and RAM. Restoring saved state +requires retaining the source's Apple platform machine identifier, but Apple +specifies undefined guest behavior when two VMs with that identifier run +concurrently. This capability remains experimental; it does not silently replace +saved RAM with a cold boot. Forked NAT connectivity is also not yet reliable. +See [VZ live-fork tests and limitations](docs/testing/vz-live-forks.md). + ### Ingress (Reverse Proxy) Create a reverse proxy from the host to your VM: diff --git a/docs/testing/vz-live-forks.md b/docs/testing/vz-live-forks.md new file mode 100644 index 000000000..e71294b08 --- /dev/null +++ b/docs/testing/vz-live-forks.md @@ -0,0 +1,77 @@ +# Experimental VZ live forks + +```sh +hypeman fork --from-running --target-state Running source copy +``` + +This suspends the source, clones its disk and saved machine state, and resumes +both guests. The fork preserves running processes and RAM rather than silently +cold-booting a disk-only copy. + +## Platform warning + +VZ saved-state restoration requires the original `VZGenericMachineIdentifier`. +Consequently, a restored Linux source and its memory-preserving fork share that +identifier. Apple's `VZGenericPlatformConfiguration.h` states that running two +VMs concurrently with the same identifier results in undefined behavior in the +guest operating system. + +Hypeman retains this experimental capability rather than rejecting the fork or +discarding saved RAM. Passing smoke tests does not establish platform support, +cryptographic safety, or reliability across macOS updates. Prefer disposable +workloads until the behavior is validated for the relevant host and guest stack. + +For an explicit disk-only copy with a fresh platform identifier: + +```sh +hypeman fork --from-running --target-state Stopped source copy +hypeman start copy +``` + +## Automated guest-state checks + +On an Apple Silicon Mac with macOS 14+, a logged-in desktop session, Go, and +Homebrew's `e2fsprogs`: + +```sh +make build-embedded sign-vz-shim +PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \ + go test -tags containers_image_openpgp \ + -run '^TestVZForkFromRunningCorrectness$' -count=3 -v \ + -timeout 10m ./lib/instances +``` + +The test is part of the normal macOS suite. It checks: + +- boot ID, workload PID, tmpfs nonce, and an increasing shell counter survive; +- both guests continue executing, with independent RAM and disk writes; +- fresh guest UUID output differs, as a smoke check rather than a randomness audit; +- the fork survives two additional standby/restore cycles while the source runs. + +`TestPrepareForkPreservesMachineIdentifier` separately verifies that fork +preparation retains the identifier required for saved-state compatibility. +Duplicate identifiers alone are not treated as an operational failure. + +## Optional network diagnostic + +The workload also serves HTTP. Enable post-fork host-to-guest HTTP checks with: + +```sh +PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \ + HYPEMAN_VZ_FORK_HTTP=1 go test -tags containers_image_openpgp \ + -run '^TestVZForkFromRunningCorrectness$' -v -timeout 10m ./lib/instances +``` + +This diagnostic is opt-in because forked VZ NAT reachability is not yet reliable. +A fork can serve HTTP over loopback and execute over vsock while the host cannot +reach its allocated IP. The guest-state assertions do not imply working network +connectivity, and this symptom has not been attributed to the shared platform ID. + +## macOS 27 boot compatibility + +The unmodified Cloud Hypervisor-derived kernel can fail before any fork on +macOS 27: its `trap_to_vmm` boot-timing MMIO write can crash the VZ service. +See [apple/container#2360](https://github.com/apple/container/issues/2360). +A failure before the source boots is not evidence of a live-fork identity bug. +Keep results using an unmodified kernel separate from experiments that remove +that hook; passing with a lab kernel does not establish stock-kernel support. diff --git a/lib/hypervisor/vz/fork_identity_darwin_test.go b/lib/hypervisor/vz/fork_identity_darwin_test.go new file mode 100644 index 000000000..986f2e9b2 --- /dev/null +++ b/lib/hypervisor/vz/fork_identity_darwin_test.go @@ -0,0 +1,55 @@ +//go:build darwin + +package vz + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + + applevz "github.com/Code-Hex/vz/v3" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/stretchr/testify/require" +) + +func TestPrepareForkPreservesMachineIdentifier(t *testing.T) { + machineID, err := applevz.NewGenericMachineIdentifier() + require.NoError(t, err) + original := shimconfig.SnapshotManifest{ + Hypervisor: string(hypervisor.TypeVZ), + MachineStateFile: shimconfig.SnapshotMachineStateFile, + ShimConfig: shimconfig.ShimConfig{ + MachineIdentifierData: base64.StdEncoding.EncodeToString(machineID.DataRepresentation()), + }, + } + data, err := json.Marshal(original) + require.NoError(t, err) + + manifestPath := filepath.Join(t.TempDir(), shimconfig.SnapshotManifestFile) + require.NoError(t, os.WriteFile(manifestPath, data, 0600)) + _, err = NewStarter().PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: manifestPath, + SourceDataDir: "/guests/source", + TargetDataDir: "/guests/fork", + }) + require.NoError(t, err) + + data, err = os.ReadFile(manifestPath) + require.NoError(t, err) + var fork shimconfig.SnapshotManifest + require.NoError(t, json.Unmarshal(data, &fork)) + + encoded, err := base64.StdEncoding.DecodeString(fork.ShimConfig.MachineIdentifierData) + require.NoError(t, err) + restoredID, err := applevz.NewGenericMachineIdentifierWithData(encoded) + require.NoError(t, err) + _, err = applevz.NewGenericPlatformConfiguration(applevz.WithGenericMachineIdentifier(restoredID)) + require.NoError(t, err) + + // Saved machine state can only be restored with its original platform identity. + require.Equal(t, machineID.DataRepresentation(), restoredID.DataRepresentation()) +} diff --git a/lib/instances/fork_correctness_darwin_test.go b/lib/instances/fork_correctness_darwin_test.go new file mode 100644 index 000000000..762f14b06 --- /dev/null +++ b/lib/instances/fork_correctness_darwin_test.go @@ -0,0 +1,228 @@ +//go:build darwin + +package instances + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/system" + "github.com/stretchr/testify/require" +) + +func TestVZForkFromRunningCorrectness(t *testing.T) { + if runtime.GOARCH != "arm64" || !isMacOS14OrLater(t) { + t.Skip("VZ running forks require Apple Silicon and macOS 14+") + } + ensureMkfsExt4Available(t) + mgr, tmpDir := setupVZTestManager(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + p := paths.New(tmpDir) + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + ref := integrationTestImageRef(t, "docker.io/library/nginx:alpine") + img, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: ref}) + require.NoError(t, err) + waitName := img.Name + if img.Digest != "" { + parsed, err := images.ParseNormalizedRef(img.Name) + require.NoError(t, err) + waitName = parsed.Repository() + "@" + img.Digest + } + require.NoError(t, imageManager.WaitForReady(ctx, waitName)) + require.NoError(t, system.NewManager(p).EnsureSystemFiles(ctx)) + + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "vz-correctness-source", + Image: ref, + Size: 2 * 1024 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeVZ, + Cmd: []string{"sh", "-ec", vzForkWorkload}, + }) + if err != nil { + dumpVZShimLogs(t, tmpDir) + require.NoError(t, err) + } + sourceID := source.Id + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, sourceID) }) + require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second)) + source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second) + require.NoError(t, err) + before := readVZForkWorkloadState(t, ctx, source) + assertVZForkHTTP(t, ctx, source, "shared") + + fork, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{ + Name: "vz-correctness-fork", + FromRunning: true, + TargetState: StateRunning, + }) + if err != nil { + dumpVZShimLogs(t, tmpDir) + require.NoError(t, err) + } + forkID := fork.Id + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, forkID) }) + require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second)) + require.NoError(t, waitForExecAgent(ctx, mgr, forkID, 60*time.Second)) + source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second) + require.NoError(t, err) + fork, err = waitForInstanceState(ctx, mgr, forkID, StateRunning, 30*time.Second) + require.NoError(t, err) + require.NotEqual(t, source.IP, fork.IP) + require.NotEqual(t, source.MAC, fork.MAC) + writeVZForkMarker(t, ctx, source, "source") + writeVZForkMarker(t, ctx, fork, "fork") + assertVZForkWorkload(t, ctx, source, fork, before) + + for i := 0; i < 2; i++ { + beforeRestore := readVZForkWorkloadState(t, ctx, fork) + _, err = mgr.StandbyInstance(ctx, forkID, StandbyInstanceRequest{}) + require.NoError(t, err) + _, err = mgr.RestoreInstance(ctx, forkID) + require.NoError(t, err) + require.NoError(t, waitForExecAgent(ctx, mgr, forkID, 60*time.Second)) + fork, err = waitForInstanceState(ctx, mgr, forkID, StateRunning, 30*time.Second) + require.NoError(t, err) + afterRestore := readVZForkWorkloadState(t, ctx, fork) + require.GreaterOrEqual(t, afterRestore.counter, beforeRestore.counter) + assertVZForkWorkload(t, ctx, source, fork, before) + } + + t.Run("host-http", func(t *testing.T) { + if os.Getenv("HYPEMAN_VZ_FORK_HTTP") != "1" { + t.Skip("set HYPEMAN_VZ_FORK_HTTP=1 to check forked NAT reachability") + } + assertVZForkHTTP(t, ctx, source, "source") + assertVZForkHTTP(t, ctx, fork, "fork") + }) +} + +// The shell counter and tmpfs nonce distinguish a RAM restore from a cold boot. +const vzForkWorkload = ` +mkdir -p /dev/shm +mount -t tmpfs -o size=16m tmpfs /dev/shm +cat /proc/sys/kernel/random/uuid > /dev/shm/fork-nonce +echo $$ > /dev/shm/fork-pid +echo shared > /dev/shm/fork-marker +echo shared > /usr/share/nginx/html/index.html +nginx -g 'daemon off;' & +i=0 +while :; do + i=$((i + 1)) + echo "$i" > /dev/shm/fork-counter.tmp + mv /dev/shm/fork-counter.tmp /dev/shm/fork-counter + sleep 0.1 +done +` + +type vzForkWorkloadState struct { + bootID string + pid string + nonce string + counter uint64 +} + +func readVZForkWorkloadState(t *testing.T, ctx context.Context, inst *Instance) vzForkWorkloadState { + t.Helper() + out, code, err := vzExecCommand(ctx, inst, "sh", "-ec", + "cat /proc/sys/kernel/random/boot_id /dev/shm/fork-pid /dev/shm/fork-nonce /dev/shm/fork-counter") + require.NoError(t, err) + require.Equal(t, 0, code, out) + fields := strings.Fields(out) + require.Len(t, fields, 4) + counter, err := strconv.ParseUint(fields[3], 10, 64) + require.NoError(t, err) + return vzForkWorkloadState{bootID: fields[0], pid: fields[1], nonce: fields[2], counter: counter} +} + +func assertVZForkWorkload(t *testing.T, ctx context.Context, source, fork *Instance, before vzForkWorkloadState) { + t.Helper() + for _, inst := range []*Instance{source, fork} { + state := readVZForkWorkloadState(t, ctx, inst) + require.Equal(t, before.bootID, state.bootID, "guest must not cold-boot") + require.Equal(t, before.pid, state.pid, "workload process must survive the fork") + require.Equal(t, before.nonce, state.nonce, "tmpfs contents must survive the fork") + require.GreaterOrEqual(t, state.counter, before.counter) + require.Eventually(t, func() bool { + out, code, err := vzExecCommand(ctx, inst, "cat", "/dev/shm/fork-counter") + counter, parseErr := strconv.ParseUint(strings.TrimSpace(out), 10, 64) + return err == nil && code == 0 && parseErr == nil && counter > state.counter + }, 10*time.Second, 100*time.Millisecond, "guest workload must continue making progress") + } + + guests := []struct { + inst *Instance + marker string + }{ + {source, "source"}, + {fork, "fork"}, + } + for _, item := range guests { + out, code, err := vzExecCommand(ctx, item.inst, "sh", "-ec", + "cat /dev/shm/fork-marker /usr/share/nginx/html/index.html") + require.NoError(t, err) + require.Equal(t, 0, code) + require.Equal(t, []string{item.marker, item.marker}, strings.Fields(out), "guest RAM and disk writes must be independent") + } + + // A smoke check for duplicated random streams, not a cryptographic audit. + sourceRandom, code, err := vzExecCommand(ctx, source, "cat", "/proc/sys/kernel/random/uuid") + require.NoError(t, err) + require.Equal(t, 0, code) + forkRandom, code, err := vzExecCommand(ctx, fork, "cat", "/proc/sys/kernel/random/uuid") + require.NoError(t, err) + require.Equal(t, 0, code) + require.NotEmpty(t, strings.TrimSpace(sourceRandom)) + require.NotEmpty(t, strings.TrimSpace(forkRandom)) + require.NotEqual(t, sourceRandom, forkRandom, "fresh random output should diverge after the fork") +} + +func writeVZForkMarker(t *testing.T, ctx context.Context, inst *Instance, marker string) { + t.Helper() + _, code, err := vzExecCommand(ctx, inst, "sh", "-ec", + fmt.Sprintf("echo %s > /dev/shm/fork-marker; echo %s > /usr/share/nginx/html/index.html", marker, marker)) + require.NoError(t, err) + require.Equal(t, 0, code) +} + +func assertVZForkHTTP(t *testing.T, ctx context.Context, inst *Instance, marker string) { + t.Helper() + client := &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{}} + defer client.CloseIdleConnections() + url := "http://" + net.JoinHostPort(inst.IP, "80") + "/" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + t.Cleanup(func() { + if t.Failed() { + diagCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + out, _, err := vzExecCommand(diagCtx, inst, "sh", "-c", "ip addr show eth0; ip route; wget -qO- http://127.0.0.1/") + t.Logf("guest HTTP diagnostics: %s (exec error: %v)", out, err) + } + }) + require.Eventually(t, func() bool { + resp, err := client.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return err == nil && resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == marker + }, 20*time.Second, 200*time.Millisecond, "host HTTP request must reach the correct guest at %s", url) +}