From ecd53bdef5b7b35053a6ee3247ecb268c9455022 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:29:36 +0000 Subject: [PATCH 1/3] Add opt-in VZ fork identity reproducers --- docs/testing/vz-machine-identity-repro.md | 81 +++++++++++++ .../vz/fork_identity_repro_darwin_test.go | 62 ++++++++++ .../fork_identity_repro_darwin_test.go | 111 ++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 docs/testing/vz-machine-identity-repro.md create mode 100644 lib/hypervisor/vz/fork_identity_repro_darwin_test.go create mode 100644 lib/instances/fork_identity_repro_darwin_test.go diff --git a/docs/testing/vz-machine-identity-repro.md b/docs/testing/vz-machine-identity-repro.md new file mode 100644 index 000000000..e06f4b692 --- /dev/null +++ b/docs/testing/vz-machine-identity-repro.md @@ -0,0 +1,81 @@ +# VZ snapshot-fork machine identity reproducers + +These opt-in diagnostics expose duplicate `VZGenericMachineIdentifier` values +without changing fork or restore behavior. They are skipped in the normal suite. + +Apple's `VZGenericPlatformConfiguration.h` requires the original machine +identifier when restoring saved state, but says concurrent VMs with the same +identifier produce undefined behavior in the guest. Changing the identifier in +a snapshot manifest is therefore not a safe fix. + +## Configuration-only reproduction + +On macOS, prepare the embedded shim: + +```sh +make sign-vz-shim +``` + +Then run: + +```sh +HYPEMAN_VZ_IDENTITY_REPRO=1 go test -tags containers_image_openpgp \ + -run '^TestPrepareForkMachineIdentifierCollisionRepro$' \ + -count=3 -v ./lib/hypervisor/vz +``` + +This creates a real VZ identifier, serializes it into a manifest, runs +`PrepareFork`, and reconstructs a VZ platform from the resulting identifier. +The final inequality assertion intentionally fails on the current implementation: +source and fork have the same platform identity. + +This does not boot a VM, prove that two VMs are running, or demonstrate any +guest-visible damage. It can run on a virtualized Mac without nested +virtualization support. + +## Live reproduction + +Use an Apple Silicon Mac running macOS 14+ that can actually launch VZ guests. +A macOS VM without nested virtualization cannot run this test. + +```sh +brew install e2fsprogs +make build-embedded sign-vz-shim +PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \ + HYPEMAN_VZ_IDENTITY_REPRO=1 go test -tags containers_image_openpgp \ + -run '^TestVZConcurrentForkMachineIdentifierRepro$' \ + -v -timeout 10m ./lib/instances +``` + +The test boots a source, puts it in standby, forks it into another standby +instance, and reads both saved identifiers before restore consumes the manifests. +It restores both guests and verifies they are running and can execute commands +before asserting that their identifiers differ. The intended diagnostic failure +is at that final assertion, not during setup or restore. + +A failure such as `Virtualization is not available on this hardware` is an +environment blocker, not a reproduction of concurrent execution. Neither test +claims that duplicate identifiers necessarily cause crashes or corruption. + +## CLI equivalent + +With a running VZ instance named `source`: + +```sh +hypeman fork --from-running --target-state Running source copy +``` + +The current implementation preserves memory in both guests and preserves their +VZ platform identifier. A collision-safe implementation must explicitly decide +whether to reject simultaneous execution or require a disk-only, cold-booted +copy. Silently throwing away saved memory would change the operation's meaning. + +The existing explicit cold-boot path is: + +```sh +hypeman fork --from-running --target-state Stopped source copy +hypeman start copy +``` + +This retains the copied disk state, not the fork's saved RAM or running processes. +The fresh VZ launch generates a new machine identifier. diff --git a/lib/hypervisor/vz/fork_identity_repro_darwin_test.go b/lib/hypervisor/vz/fork_identity_repro_darwin_test.go new file mode 100644 index 000000000..18de8da68 --- /dev/null +++ b/lib/hypervisor/vz/fork_identity_repro_darwin_test.go @@ -0,0 +1,62 @@ +//go:build darwin + +package vz + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + + applevz "github.com/Code-Hex/vz/v3" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/stretchr/testify/require" +) + +// TestPrepareForkMachineIdentifierCollisionRepro intentionally fails when enabled. +// It diagnoses duplicated platform identity, not guest-visible consequences. +func TestPrepareForkMachineIdentifierCollisionRepro(t *testing.T) { + if os.Getenv("HYPEMAN_VZ_IDENTITY_REPRO") != "1" { + t.Skip("set HYPEMAN_VZ_IDENTITY_REPRO=1 to reproduce duplicate VZ machine identifiers") + } + + machineID, err := applevz.NewGenericMachineIdentifier() + require.NoError(t, err) + original := shimconfig.SnapshotManifest{ + Hypervisor: string(hypervisor.TypeVZ), + MachineStateFile: shimconfig.SnapshotMachineStateFile, + ShimConfig: shimconfig.ShimConfig{ + MachineIdentifierData: base64.StdEncoding.EncodeToString(machineID.DataRepresentation()), + }, + } + data, err := json.Marshal(original) + require.NoError(t, err) + + manifestPath := filepath.Join(t.TempDir(), shimconfig.SnapshotManifestFile) + require.NoError(t, os.WriteFile(manifestPath, data, 0600)) + _, err = NewStarter().PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{ + SnapshotConfigPath: manifestPath, + SourceDataDir: "/guests/source", + TargetDataDir: "/guests/fork", + }) + require.NoError(t, err) + + data, err = os.ReadFile(manifestPath) + require.NoError(t, err) + var fork shimconfig.SnapshotManifest + require.NoError(t, json.Unmarshal(data, &fork)) + + encoded, err := base64.StdEncoding.DecodeString(fork.ShimConfig.MachineIdentifierData) + require.NoError(t, err) + restoredID, err := applevz.NewGenericMachineIdentifierWithData(encoded) + require.NoError(t, err) + _, err = applevz.NewGenericPlatformConfiguration(applevz.WithGenericMachineIdentifier(restoredID)) + require.NoError(t, err) + + t.Log("fork manifest reconstructs a VZ platform; no VM was booted") + require.NotEqual(t, machineID.DataRepresentation(), restoredID.DataRepresentation(), + "source and fork reconstruct the same platform identifier; concurrent execution needs an explicit identity-collision policy") +} diff --git a/lib/instances/fork_identity_repro_darwin_test.go b/lib/instances/fork_identity_repro_darwin_test.go new file mode 100644 index 000000000..3ba891b03 --- /dev/null +++ b/lib/instances/fork_identity_repro_darwin_test.go @@ -0,0 +1,111 @@ +//go:build darwin + +package instances + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/system" + "github.com/stretchr/testify/require" +) + +// TestVZConcurrentForkMachineIdentifierRepro checks identity after both guests +// resume and execute commands. It does not assert that duplicate IDs cause crashes. +func TestVZConcurrentForkMachineIdentifierRepro(t *testing.T) { + if os.Getenv("HYPEMAN_VZ_IDENTITY_REPRO") != "1" { + t.Skip("set HYPEMAN_VZ_IDENTITY_REPRO=1 to reproduce concurrent VZ identity collisions") + } + if runtime.GOARCH != "arm64" || !isMacOS14OrLater(t) { + t.Skip("VZ standby/restore requires Apple Silicon and macOS 14+") + } + ensureMkfsExt4Available(t) + + mgr, tmpDir := setupVZTestManager(t) + p := paths.New(tmpDir) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + ref := integrationTestImageRef(t, "docker.io/library/alpine:latest") + img, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: ref}) + require.NoError(t, err) + waitName := img.Name + if img.Digest != "" { + parsed, err := images.ParseNormalizedRef(img.Name) + require.NoError(t, err) + waitName = parsed.Repository() + "@" + img.Digest + } + require.NoError(t, imageManager.WaitForReady(ctx, waitName)) + require.NoError(t, system.NewManager(p).EnsureSystemFiles(ctx)) + + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "vz-identity-source", + Image: ref, + Size: 2 * 1024 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Hypervisor: hypervisor.TypeVZ, + Cmd: []string{"sleep", "infinity"}, + }) + if err != nil { + dumpVZShimLogs(t, tmpDir) + require.NoError(t, err) + } + sourceID := source.Id + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, sourceID) }) + require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second)) + _, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second) + require.NoError(t, err) + _, err = mgr.StandbyInstance(ctx, sourceID, StandbyInstanceRequest{}) + require.NoError(t, err) + sourceMachineID := readVZReproMachineIdentifier(t, p.InstanceSnapshotLatest(sourceID)) + + fork, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{ + Name: "vz-identity-fork", + TargetState: StateStandby, + }) + require.NoError(t, err) + forkID := fork.Id + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, forkID) }) + forkMachineID := readVZReproMachineIdentifier(t, p.InstanceSnapshotLatest(forkID)) + + for _, id := range []string{sourceID, forkID} { + _, err = mgr.RestoreInstance(ctx, id) + require.NoError(t, err) + require.NoError(t, waitForExecAgent(ctx, mgr, id, 60*time.Second)) + } + for _, id := range []string{sourceID, forkID} { + inst, err := waitForInstanceState(ctx, mgr, id, StateRunning, 30*time.Second) + require.NoError(t, err) + out, code, err := vzExecCommand(ctx, inst, "echo", "identity-repro-running") + require.NoError(t, err) + require.Equal(t, 0, code) + require.Equal(t, "identity-repro-running", strings.TrimSpace(out)) + } + + t.Log("source and fork are both running and can execute guest commands") + require.NotEqual(t, sourceMachineID, forkMachineID, + "two running VZ guests were restored with the same platform identifier") +} + +func readVZReproMachineIdentifier(t *testing.T, snapshotDir string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(snapshotDir, shimconfig.SnapshotManifestFile)) + require.NoError(t, err) + var manifest shimconfig.SnapshotManifest + require.NoError(t, json.Unmarshal(data, &manifest)) + require.NotEmpty(t, manifest.ShimConfig.MachineIdentifierData) + return manifest.ShimConfig.MachineIdentifierData +} From 74af3aeaa2489151ab6dbc32d6acf9de80fbf69f Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:03:55 +0000 Subject: [PATCH 2/3] Document experimental VZ forks and test guest continuity --- README.md | 13 + docs/testing/vz-live-forks.md | 77 ++++++ docs/testing/vz-machine-identity-repro.md | 81 ------- ...n_test.go => fork_identity_darwin_test.go} | 13 +- lib/instances/fork_correctness_darwin_test.go | 228 ++++++++++++++++++ .../fork_identity_repro_darwin_test.go | 111 --------- 6 files changed, 321 insertions(+), 202 deletions(-) create mode 100644 docs/testing/vz-live-forks.md delete mode 100644 docs/testing/vz-machine-identity-repro.md rename lib/hypervisor/vz/{fork_identity_repro_darwin_test.go => fork_identity_darwin_test.go} (70%) create mode 100644 lib/instances/fork_correctness_darwin_test.go delete mode 100644 lib/instances/fork_identity_repro_darwin_test.go diff --git a/README.md b/README.md index 70d3ee66f..88dd98ec6 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,19 @@ hypeman restore my-app hypeman rm --force --all ``` +### Experimental live forks on macOS + +```bash +hypeman fork --from-running --target-state Running source copy +``` + +VZ live forks preserve running Linux processes and RAM. Restoring saved state +requires retaining the source's Apple platform machine identifier, but Apple +specifies undefined guest behavior when two VMs with that identifier run +concurrently. This capability remains experimental; it does not silently replace +saved RAM with a cold boot. Forked NAT connectivity is also not yet reliable. +See [VZ live-fork tests and limitations](docs/testing/vz-live-forks.md). + ### Ingress (Reverse Proxy) Create a reverse proxy from the host to your VM: diff --git a/docs/testing/vz-live-forks.md b/docs/testing/vz-live-forks.md new file mode 100644 index 000000000..e71294b08 --- /dev/null +++ b/docs/testing/vz-live-forks.md @@ -0,0 +1,77 @@ +# Experimental VZ live forks + +```sh +hypeman fork --from-running --target-state Running source copy +``` + +This suspends the source, clones its disk and saved machine state, and resumes +both guests. The fork preserves running processes and RAM rather than silently +cold-booting a disk-only copy. + +## Platform warning + +VZ saved-state restoration requires the original `VZGenericMachineIdentifier`. +Consequently, a restored Linux source and its memory-preserving fork share that +identifier. Apple's `VZGenericPlatformConfiguration.h` states that running two +VMs concurrently with the same identifier results in undefined behavior in the +guest operating system. + +Hypeman retains this experimental capability rather than rejecting the fork or +discarding saved RAM. Passing smoke tests does not establish platform support, +cryptographic safety, or reliability across macOS updates. Prefer disposable +workloads until the behavior is validated for the relevant host and guest stack. + +For an explicit disk-only copy with a fresh platform identifier: + +```sh +hypeman fork --from-running --target-state Stopped source copy +hypeman start copy +``` + +## Automated guest-state checks + +On an Apple Silicon Mac with macOS 14+, a logged-in desktop session, Go, and +Homebrew's `e2fsprogs`: + +```sh +make build-embedded sign-vz-shim +PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \ + go test -tags containers_image_openpgp \ + -run '^TestVZForkFromRunningCorrectness$' -count=3 -v \ + -timeout 10m ./lib/instances +``` + +The test is part of the normal macOS suite. It checks: + +- boot ID, workload PID, tmpfs nonce, and an increasing shell counter survive; +- both guests continue executing, with independent RAM and disk writes; +- fresh guest UUID output differs, as a smoke check rather than a randomness audit; +- the fork survives two additional standby/restore cycles while the source runs. + +`TestPrepareForkPreservesMachineIdentifier` separately verifies that fork +preparation retains the identifier required for saved-state compatibility. +Duplicate identifiers alone are not treated as an operational failure. + +## Optional network diagnostic + +The workload also serves HTTP. Enable post-fork host-to-guest HTTP checks with: + +```sh +PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \ + HYPEMAN_VZ_FORK_HTTP=1 go test -tags containers_image_openpgp \ + -run '^TestVZForkFromRunningCorrectness$' -v -timeout 10m ./lib/instances +``` + +This diagnostic is opt-in because forked VZ NAT reachability is not yet reliable. +A fork can serve HTTP over loopback and execute over vsock while the host cannot +reach its allocated IP. The guest-state assertions do not imply working network +connectivity, and this symptom has not been attributed to the shared platform ID. + +## macOS 27 boot compatibility + +The unmodified Cloud Hypervisor-derived kernel can fail before any fork on +macOS 27: its `trap_to_vmm` boot-timing MMIO write can crash the VZ service. +See [apple/container#2360](https://github.com/apple/container/issues/2360). +A failure before the source boots is not evidence of a live-fork identity bug. +Keep results using an unmodified kernel separate from experiments that remove +that hook; passing with a lab kernel does not establish stock-kernel support. diff --git a/docs/testing/vz-machine-identity-repro.md b/docs/testing/vz-machine-identity-repro.md deleted file mode 100644 index e06f4b692..000000000 --- a/docs/testing/vz-machine-identity-repro.md +++ /dev/null @@ -1,81 +0,0 @@ -# VZ snapshot-fork machine identity reproducers - -These opt-in diagnostics expose duplicate `VZGenericMachineIdentifier` values -without changing fork or restore behavior. They are skipped in the normal suite. - -Apple's `VZGenericPlatformConfiguration.h` requires the original machine -identifier when restoring saved state, but says concurrent VMs with the same -identifier produce undefined behavior in the guest. Changing the identifier in -a snapshot manifest is therefore not a safe fix. - -## Configuration-only reproduction - -On macOS, prepare the embedded shim: - -```sh -make sign-vz-shim -``` - -Then run: - -```sh -HYPEMAN_VZ_IDENTITY_REPRO=1 go test -tags containers_image_openpgp \ - -run '^TestPrepareForkMachineIdentifierCollisionRepro$' \ - -count=3 -v ./lib/hypervisor/vz -``` - -This creates a real VZ identifier, serializes it into a manifest, runs -`PrepareFork`, and reconstructs a VZ platform from the resulting identifier. -The final inequality assertion intentionally fails on the current implementation: -source and fork have the same platform identity. - -This does not boot a VM, prove that two VMs are running, or demonstrate any -guest-visible damage. It can run on a virtualized Mac without nested -virtualization support. - -## Live reproduction - -Use an Apple Silicon Mac running macOS 14+ that can actually launch VZ guests. -A macOS VM without nested virtualization cannot run this test. - -```sh -brew install e2fsprogs -make build-embedded sign-vz-shim -PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \ - HYPEMAN_VZ_IDENTITY_REPRO=1 go test -tags containers_image_openpgp \ - -run '^TestVZConcurrentForkMachineIdentifierRepro$' \ - -v -timeout 10m ./lib/instances -``` - -The test boots a source, puts it in standby, forks it into another standby -instance, and reads both saved identifiers before restore consumes the manifests. -It restores both guests and verifies they are running and can execute commands -before asserting that their identifiers differ. The intended diagnostic failure -is at that final assertion, not during setup or restore. - -A failure such as `Virtualization is not available on this hardware` is an -environment blocker, not a reproduction of concurrent execution. Neither test -claims that duplicate identifiers necessarily cause crashes or corruption. - -## CLI equivalent - -With a running VZ instance named `source`: - -```sh -hypeman fork --from-running --target-state Running source copy -``` - -The current implementation preserves memory in both guests and preserves their -VZ platform identifier. A collision-safe implementation must explicitly decide -whether to reject simultaneous execution or require a disk-only, cold-booted -copy. Silently throwing away saved memory would change the operation's meaning. - -The existing explicit cold-boot path is: - -```sh -hypeman fork --from-running --target-state Stopped source copy -hypeman start copy -``` - -This retains the copied disk state, not the fork's saved RAM or running processes. -The fresh VZ launch generates a new machine identifier. diff --git a/lib/hypervisor/vz/fork_identity_repro_darwin_test.go b/lib/hypervisor/vz/fork_identity_darwin_test.go similarity index 70% rename from lib/hypervisor/vz/fork_identity_repro_darwin_test.go rename to lib/hypervisor/vz/fork_identity_darwin_test.go index 18de8da68..986f2e9b2 100644 --- a/lib/hypervisor/vz/fork_identity_repro_darwin_test.go +++ b/lib/hypervisor/vz/fork_identity_darwin_test.go @@ -16,13 +16,7 @@ import ( "github.com/stretchr/testify/require" ) -// TestPrepareForkMachineIdentifierCollisionRepro intentionally fails when enabled. -// It diagnoses duplicated platform identity, not guest-visible consequences. -func TestPrepareForkMachineIdentifierCollisionRepro(t *testing.T) { - if os.Getenv("HYPEMAN_VZ_IDENTITY_REPRO") != "1" { - t.Skip("set HYPEMAN_VZ_IDENTITY_REPRO=1 to reproduce duplicate VZ machine identifiers") - } - +func TestPrepareForkPreservesMachineIdentifier(t *testing.T) { machineID, err := applevz.NewGenericMachineIdentifier() require.NoError(t, err) original := shimconfig.SnapshotManifest{ @@ -56,7 +50,6 @@ func TestPrepareForkMachineIdentifierCollisionRepro(t *testing.T) { _, err = applevz.NewGenericPlatformConfiguration(applevz.WithGenericMachineIdentifier(restoredID)) require.NoError(t, err) - t.Log("fork manifest reconstructs a VZ platform; no VM was booted") - require.NotEqual(t, machineID.DataRepresentation(), restoredID.DataRepresentation(), - "source and fork reconstruct the same platform identifier; concurrent execution needs an explicit identity-collision policy") + // Saved machine state can only be restored with its original platform identity. + require.Equal(t, machineID.DataRepresentation(), restoredID.DataRepresentation()) } diff --git a/lib/instances/fork_correctness_darwin_test.go b/lib/instances/fork_correctness_darwin_test.go new file mode 100644 index 000000000..762f14b06 --- /dev/null +++ b/lib/instances/fork_correctness_darwin_test.go @@ -0,0 +1,228 @@ +//go:build darwin + +package instances + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/system" + "github.com/stretchr/testify/require" +) + +func TestVZForkFromRunningCorrectness(t *testing.T) { + if runtime.GOARCH != "arm64" || !isMacOS14OrLater(t) { + t.Skip("VZ running forks require Apple Silicon and macOS 14+") + } + ensureMkfsExt4Available(t) + mgr, tmpDir := setupVZTestManager(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + p := paths.New(tmpDir) + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + ref := integrationTestImageRef(t, "docker.io/library/nginx:alpine") + img, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: ref}) + require.NoError(t, err) + waitName := img.Name + if img.Digest != "" { + parsed, err := images.ParseNormalizedRef(img.Name) + require.NoError(t, err) + waitName = parsed.Repository() + "@" + img.Digest + } + require.NoError(t, imageManager.WaitForReady(ctx, waitName)) + require.NoError(t, system.NewManager(p).EnsureSystemFiles(ctx)) + + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "vz-correctness-source", + Image: ref, + Size: 2 * 1024 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeVZ, + Cmd: []string{"sh", "-ec", vzForkWorkload}, + }) + if err != nil { + dumpVZShimLogs(t, tmpDir) + require.NoError(t, err) + } + sourceID := source.Id + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, sourceID) }) + require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second)) + source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second) + require.NoError(t, err) + before := readVZForkWorkloadState(t, ctx, source) + assertVZForkHTTP(t, ctx, source, "shared") + + fork, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{ + Name: "vz-correctness-fork", + FromRunning: true, + TargetState: StateRunning, + }) + if err != nil { + dumpVZShimLogs(t, tmpDir) + require.NoError(t, err) + } + forkID := fork.Id + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, forkID) }) + require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second)) + require.NoError(t, waitForExecAgent(ctx, mgr, forkID, 60*time.Second)) + source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second) + require.NoError(t, err) + fork, err = waitForInstanceState(ctx, mgr, forkID, StateRunning, 30*time.Second) + require.NoError(t, err) + require.NotEqual(t, source.IP, fork.IP) + require.NotEqual(t, source.MAC, fork.MAC) + writeVZForkMarker(t, ctx, source, "source") + writeVZForkMarker(t, ctx, fork, "fork") + assertVZForkWorkload(t, ctx, source, fork, before) + + for i := 0; i < 2; i++ { + beforeRestore := readVZForkWorkloadState(t, ctx, fork) + _, err = mgr.StandbyInstance(ctx, forkID, StandbyInstanceRequest{}) + require.NoError(t, err) + _, err = mgr.RestoreInstance(ctx, forkID) + require.NoError(t, err) + require.NoError(t, waitForExecAgent(ctx, mgr, forkID, 60*time.Second)) + fork, err = waitForInstanceState(ctx, mgr, forkID, StateRunning, 30*time.Second) + require.NoError(t, err) + afterRestore := readVZForkWorkloadState(t, ctx, fork) + require.GreaterOrEqual(t, afterRestore.counter, beforeRestore.counter) + assertVZForkWorkload(t, ctx, source, fork, before) + } + + t.Run("host-http", func(t *testing.T) { + if os.Getenv("HYPEMAN_VZ_FORK_HTTP") != "1" { + t.Skip("set HYPEMAN_VZ_FORK_HTTP=1 to check forked NAT reachability") + } + assertVZForkHTTP(t, ctx, source, "source") + assertVZForkHTTP(t, ctx, fork, "fork") + }) +} + +// The shell counter and tmpfs nonce distinguish a RAM restore from a cold boot. +const vzForkWorkload = ` +mkdir -p /dev/shm +mount -t tmpfs -o size=16m tmpfs /dev/shm +cat /proc/sys/kernel/random/uuid > /dev/shm/fork-nonce +echo $$ > /dev/shm/fork-pid +echo shared > /dev/shm/fork-marker +echo shared > /usr/share/nginx/html/index.html +nginx -g 'daemon off;' & +i=0 +while :; do + i=$((i + 1)) + echo "$i" > /dev/shm/fork-counter.tmp + mv /dev/shm/fork-counter.tmp /dev/shm/fork-counter + sleep 0.1 +done +` + +type vzForkWorkloadState struct { + bootID string + pid string + nonce string + counter uint64 +} + +func readVZForkWorkloadState(t *testing.T, ctx context.Context, inst *Instance) vzForkWorkloadState { + t.Helper() + out, code, err := vzExecCommand(ctx, inst, "sh", "-ec", + "cat /proc/sys/kernel/random/boot_id /dev/shm/fork-pid /dev/shm/fork-nonce /dev/shm/fork-counter") + require.NoError(t, err) + require.Equal(t, 0, code, out) + fields := strings.Fields(out) + require.Len(t, fields, 4) + counter, err := strconv.ParseUint(fields[3], 10, 64) + require.NoError(t, err) + return vzForkWorkloadState{bootID: fields[0], pid: fields[1], nonce: fields[2], counter: counter} +} + +func assertVZForkWorkload(t *testing.T, ctx context.Context, source, fork *Instance, before vzForkWorkloadState) { + t.Helper() + for _, inst := range []*Instance{source, fork} { + state := readVZForkWorkloadState(t, ctx, inst) + require.Equal(t, before.bootID, state.bootID, "guest must not cold-boot") + require.Equal(t, before.pid, state.pid, "workload process must survive the fork") + require.Equal(t, before.nonce, state.nonce, "tmpfs contents must survive the fork") + require.GreaterOrEqual(t, state.counter, before.counter) + require.Eventually(t, func() bool { + out, code, err := vzExecCommand(ctx, inst, "cat", "/dev/shm/fork-counter") + counter, parseErr := strconv.ParseUint(strings.TrimSpace(out), 10, 64) + return err == nil && code == 0 && parseErr == nil && counter > state.counter + }, 10*time.Second, 100*time.Millisecond, "guest workload must continue making progress") + } + + guests := []struct { + inst *Instance + marker string + }{ + {source, "source"}, + {fork, "fork"}, + } + for _, item := range guests { + out, code, err := vzExecCommand(ctx, item.inst, "sh", "-ec", + "cat /dev/shm/fork-marker /usr/share/nginx/html/index.html") + require.NoError(t, err) + require.Equal(t, 0, code) + require.Equal(t, []string{item.marker, item.marker}, strings.Fields(out), "guest RAM and disk writes must be independent") + } + + // A smoke check for duplicated random streams, not a cryptographic audit. + sourceRandom, code, err := vzExecCommand(ctx, source, "cat", "/proc/sys/kernel/random/uuid") + require.NoError(t, err) + require.Equal(t, 0, code) + forkRandom, code, err := vzExecCommand(ctx, fork, "cat", "/proc/sys/kernel/random/uuid") + require.NoError(t, err) + require.Equal(t, 0, code) + require.NotEmpty(t, strings.TrimSpace(sourceRandom)) + require.NotEmpty(t, strings.TrimSpace(forkRandom)) + require.NotEqual(t, sourceRandom, forkRandom, "fresh random output should diverge after the fork") +} + +func writeVZForkMarker(t *testing.T, ctx context.Context, inst *Instance, marker string) { + t.Helper() + _, code, err := vzExecCommand(ctx, inst, "sh", "-ec", + fmt.Sprintf("echo %s > /dev/shm/fork-marker; echo %s > /usr/share/nginx/html/index.html", marker, marker)) + require.NoError(t, err) + require.Equal(t, 0, code) +} + +func assertVZForkHTTP(t *testing.T, ctx context.Context, inst *Instance, marker string) { + t.Helper() + client := &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{}} + defer client.CloseIdleConnections() + url := "http://" + net.JoinHostPort(inst.IP, "80") + "/" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + t.Cleanup(func() { + if t.Failed() { + diagCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + out, _, err := vzExecCommand(diagCtx, inst, "sh", "-c", "ip addr show eth0; ip route; wget -qO- http://127.0.0.1/") + t.Logf("guest HTTP diagnostics: %s (exec error: %v)", out, err) + } + }) + require.Eventually(t, func() bool { + resp, err := client.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return err == nil && resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == marker + }, 20*time.Second, 200*time.Millisecond, "host HTTP request must reach the correct guest at %s", url) +} diff --git a/lib/instances/fork_identity_repro_darwin_test.go b/lib/instances/fork_identity_repro_darwin_test.go deleted file mode 100644 index 3ba891b03..000000000 --- a/lib/instances/fork_identity_repro_darwin_test.go +++ /dev/null @@ -1,111 +0,0 @@ -//go:build darwin - -package instances - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "runtime" - "strings" - "testing" - "time" - - "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/paths" - "github.com/kernel/hypeman/lib/system" - "github.com/stretchr/testify/require" -) - -// TestVZConcurrentForkMachineIdentifierRepro checks identity after both guests -// resume and execute commands. It does not assert that duplicate IDs cause crashes. -func TestVZConcurrentForkMachineIdentifierRepro(t *testing.T) { - if os.Getenv("HYPEMAN_VZ_IDENTITY_REPRO") != "1" { - t.Skip("set HYPEMAN_VZ_IDENTITY_REPRO=1 to reproduce concurrent VZ identity collisions") - } - if runtime.GOARCH != "arm64" || !isMacOS14OrLater(t) { - t.Skip("VZ standby/restore requires Apple Silicon and macOS 14+") - } - ensureMkfsExt4Available(t) - - mgr, tmpDir := setupVZTestManager(t) - p := paths.New(tmpDir) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - imageManager, err := images.NewManager(p, 1, nil) - require.NoError(t, err) - ref := integrationTestImageRef(t, "docker.io/library/alpine:latest") - img, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: ref}) - require.NoError(t, err) - waitName := img.Name - if img.Digest != "" { - parsed, err := images.ParseNormalizedRef(img.Name) - require.NoError(t, err) - waitName = parsed.Repository() + "@" + img.Digest - } - require.NoError(t, imageManager.WaitForReady(ctx, waitName)) - require.NoError(t, system.NewManager(p).EnsureSystemFiles(ctx)) - - source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ - Name: "vz-identity-source", - Image: ref, - Size: 2 * 1024 * 1024 * 1024, - OverlaySize: 10 * 1024 * 1024 * 1024, - Vcpus: 1, - NetworkEnabled: false, - Hypervisor: hypervisor.TypeVZ, - Cmd: []string{"sleep", "infinity"}, - }) - if err != nil { - dumpVZShimLogs(t, tmpDir) - require.NoError(t, err) - } - sourceID := source.Id - t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, sourceID) }) - require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second)) - _, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second) - require.NoError(t, err) - _, err = mgr.StandbyInstance(ctx, sourceID, StandbyInstanceRequest{}) - require.NoError(t, err) - sourceMachineID := readVZReproMachineIdentifier(t, p.InstanceSnapshotLatest(sourceID)) - - fork, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{ - Name: "vz-identity-fork", - TargetState: StateStandby, - }) - require.NoError(t, err) - forkID := fork.Id - t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, forkID) }) - forkMachineID := readVZReproMachineIdentifier(t, p.InstanceSnapshotLatest(forkID)) - - for _, id := range []string{sourceID, forkID} { - _, err = mgr.RestoreInstance(ctx, id) - require.NoError(t, err) - require.NoError(t, waitForExecAgent(ctx, mgr, id, 60*time.Second)) - } - for _, id := range []string{sourceID, forkID} { - inst, err := waitForInstanceState(ctx, mgr, id, StateRunning, 30*time.Second) - require.NoError(t, err) - out, code, err := vzExecCommand(ctx, inst, "echo", "identity-repro-running") - require.NoError(t, err) - require.Equal(t, 0, code) - require.Equal(t, "identity-repro-running", strings.TrimSpace(out)) - } - - t.Log("source and fork are both running and can execute guest commands") - require.NotEqual(t, sourceMachineID, forkMachineID, - "two running VZ guests were restored with the same platform identifier") -} - -func readVZReproMachineIdentifier(t *testing.T, snapshotDir string) string { - t.Helper() - data, err := os.ReadFile(filepath.Join(snapshotDir, shimconfig.SnapshotManifestFile)) - require.NoError(t, err) - var manifest shimconfig.SnapshotManifest - require.NoError(t, json.Unmarshal(data, &manifest)) - require.NotEmpty(t, manifest.ShimConfig.MachineIdentifierData) - return manifest.ShimConfig.MachineIdentifierData -} From fbf5c877f3741b57f9a54b1945c14f73f00d4a64 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:22:53 +0000 Subject: [PATCH 3/3] Record the macOS host version in CI --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bdafcd13a..ac122ce18 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -395,6 +395,10 @@ jobs: env: DATA_DIR: /tmp/hypeman-ci-${{ github.run_id }} steps: + - name: Report macOS host version + run: | + sw_vers + sysctl -n machdep.cpu.brand_string - uses: actions/checkout@v4 with: repository: ${{ env.TEST_SOURCE_REPO }}