Document experimental VZ live forks and test guest state

.github/workflows/test.yml

@@ -395,6 +395,10 @@ jobs:
     env:
       DATA_DIR: /tmp/hypeman-ci-${{ github.run_id }}
     steps:
+      - name: Report macOS host version
+        run: |
+          sw_vers
+          sysctl -n machdep.cpu.brand_string
       - uses: actions/checkout@v4
         with:
           repository: ${{ env.TEST_SOURCE_REPO }}

README.md

@@ -238,6 +238,19 @@ hypeman restore my-app
 hypeman rm --force --all
 ```
 
+### Experimental live forks on macOS
+
+```bash
+hypeman fork --from-running --target-state Running source copy
+```
+
+VZ live forks preserve running Linux processes and RAM. Restoring saved state
+requires retaining the source's Apple platform machine identifier, but Apple
+specifies undefined guest behavior when two VMs with that identifier run
+concurrently. This capability remains experimental; it does not silently replace
+saved RAM with a cold boot. Forked NAT connectivity is also not yet reliable.
+See [VZ live-fork tests and limitations](docs/testing/vz-live-forks.md).
+
 ### Ingress (Reverse Proxy)
 
 Create a reverse proxy from the host to your VM:

docs/testing/vz-live-forks.md

@@ -0,0 +1,77 @@
+# Experimental VZ live forks
+
+```sh
+hypeman fork --from-running --target-state Running source copy
+```
+
+This suspends the source, clones its disk and saved machine state, and resumes
+both guests. The fork preserves running processes and RAM rather than silently
+cold-booting a disk-only copy.
+
+## Platform warning
+
+VZ saved-state restoration requires the original `VZGenericMachineIdentifier`.
+Consequently, a restored Linux source and its memory-preserving fork share that
+identifier. Apple's `VZGenericPlatformConfiguration.h` states that running two
+VMs concurrently with the same identifier results in undefined behavior in the
+guest operating system.
+
+Hypeman retains this experimental capability rather than rejecting the fork or
+discarding saved RAM. Passing smoke tests does not establish platform support,
+cryptographic safety, or reliability across macOS updates. Prefer disposable
+workloads until the behavior is validated for the relevant host and guest stack.
+
+For an explicit disk-only copy with a fresh platform identifier:
+
+```sh
+hypeman fork --from-running --target-state Stopped source copy
+hypeman start copy
+```
+
+## Automated guest-state checks
+
+On an Apple Silicon Mac with macOS 14+, a logged-in desktop session, Go, and
+Homebrew's `e2fsprogs`:
+
+```sh
+make build-embedded sign-vz-shim
+PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \
+  go test -tags containers_image_openpgp \
+  -run '^TestVZForkFromRunningCorrectness$' -count=3 -v \
+  -timeout 10m ./lib/instances
+```
+
+The test is part of the normal macOS suite. It checks:
+
+- boot ID, workload PID, tmpfs nonce, and an increasing shell counter survive;
+- both guests continue executing, with independent RAM and disk writes;
+- fresh guest UUID output differs, as a smoke check rather than a randomness audit;
+- the fork survives two additional standby/restore cycles while the source runs.
+
+`TestPrepareForkPreservesMachineIdentifier` separately verifies that fork
+preparation retains the identifier required for saved-state compatibility.
+Duplicate identifiers alone are not treated as an operational failure.
+
+## Optional network diagnostic
+
+The workload also serves HTTP. Enable post-fork host-to-guest HTTP checks with:
+
+```sh
+PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \
+  HYPEMAN_VZ_FORK_HTTP=1 go test -tags containers_image_openpgp \
+  -run '^TestVZForkFromRunningCorrectness$' -v -timeout 10m ./lib/instances
+```
+
+This diagnostic is opt-in because forked VZ NAT reachability is not yet reliable.
+A fork can serve HTTP over loopback and execute over vsock while the host cannot
+reach its allocated IP. The guest-state assertions do not imply working network
+connectivity, and this symptom has not been attributed to the shared platform ID.
+
+## macOS 27 boot compatibility
+
+The unmodified Cloud Hypervisor-derived kernel can fail before any fork on
+macOS 27: its `trap_to_vmm` boot-timing MMIO write can crash the VZ service.
+See [apple/container#2360](https://github.com/apple/container/issues/2360).
+A failure before the source boots is not evidence of a live-fork identity bug.
+Keep results using an unmodified kernel separate from experiments that remove
+that hook; passing with a lab kernel does not establish stock-kernel support.

lib/hypervisor/vz/fork_identity_darwin_test.go

@@ -0,0 +1,55 @@
+//go:build darwin
+
+package vz
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	applevz "github.com/Code-Hex/vz/v3"
+	"github.com/kernel/hypeman/lib/hypervisor"
+	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrepareForkPreservesMachineIdentifier(t *testing.T) {
+	machineID, err := applevz.NewGenericMachineIdentifier()
+	require.NoError(t, err)
+	original := shimconfig.SnapshotManifest{
+		Hypervisor:       string(hypervisor.TypeVZ),
+		MachineStateFile: shimconfig.SnapshotMachineStateFile,
+		ShimConfig: shimconfig.ShimConfig{
+			MachineIdentifierData: base64.StdEncoding.EncodeToString(machineID.DataRepresentation()),
+		},
+	}
+	data, err := json.Marshal(original)
+	require.NoError(t, err)
+
+	manifestPath := filepath.Join(t.TempDir(), shimconfig.SnapshotManifestFile)
+	require.NoError(t, os.WriteFile(manifestPath, data, 0600))
+	_, err = NewStarter().PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{
+		SnapshotConfigPath: manifestPath,
+		SourceDataDir:      "/guests/source",
+		TargetDataDir:      "/guests/fork",
+	})
+	require.NoError(t, err)
+
+	data, err = os.ReadFile(manifestPath)
+	require.NoError(t, err)
+	var fork shimconfig.SnapshotManifest
+	require.NoError(t, json.Unmarshal(data, &fork))
+
+	encoded, err := base64.StdEncoding.DecodeString(fork.ShimConfig.MachineIdentifierData)
+	require.NoError(t, err)
+	restoredID, err := applevz.NewGenericMachineIdentifierWithData(encoded)
+	require.NoError(t, err)
+	_, err = applevz.NewGenericPlatformConfiguration(applevz.WithGenericMachineIdentifier(restoredID))
+	require.NoError(t, err)
+
+	// Saved machine state can only be restored with its original platform identity.
+	require.Equal(t, machineID.DataRepresentation(), restoredID.DataRepresentation())
+}

lib/instances/fork_correctness_darwin_test.go

@@ -0,0 +1,228 @@
+//go:build darwin
+
+package instances
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kernel/hypeman/lib/hypervisor"
+	"github.com/kernel/hypeman/lib/images"
+	"github.com/kernel/hypeman/lib/paths"
+	"github.com/kernel/hypeman/lib/system"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVZForkFromRunningCorrectness(t *testing.T) {
+	if runtime.GOARCH != "arm64" || !isMacOS14OrLater(t) {
+		t.Skip("VZ running forks require Apple Silicon and macOS 14+")
+	}
+	ensureMkfsExt4Available(t)
+	mgr, tmpDir := setupVZTestManager(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	p := paths.New(tmpDir)
+	imageManager, err := images.NewManager(p, 1, nil)
+	require.NoError(t, err)
+	ref := integrationTestImageRef(t, "docker.io/library/nginx:alpine")
+	img, err := imageManager.CreateImage(ctx, images.CreateImageRequest{Name: ref})
+	require.NoError(t, err)
+	waitName := img.Name
+	if img.Digest != "" {
+		parsed, err := images.ParseNormalizedRef(img.Name)
+		require.NoError(t, err)
+		waitName = parsed.Repository() + "@" + img.Digest
+	}
+	require.NoError(t, imageManager.WaitForReady(ctx, waitName))
+	require.NoError(t, system.NewManager(p).EnsureSystemFiles(ctx))
+
+	source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{
+		Name:           "vz-correctness-source",
+		Image:          ref,
+		Size:           2 * 1024 * 1024 * 1024,
+		OverlaySize:    10 * 1024 * 1024 * 1024,
+		Vcpus:          1,
+		NetworkEnabled: true,
+		Hypervisor:     hypervisor.TypeVZ,
+		Cmd:            []string{"sh", "-ec", vzForkWorkload},
+	})
+	if err != nil {
+		dumpVZShimLogs(t, tmpDir)
+		require.NoError(t, err)
+	}
+	sourceID := source.Id
+	t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, sourceID) })
+	require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second))
+	source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second)
+	require.NoError(t, err)
+	before := readVZForkWorkloadState(t, ctx, source)
+	assertVZForkHTTP(t, ctx, source, "shared")
+
+	fork, err := mgr.ForkInstance(ctx, sourceID, ForkInstanceRequest{
+		Name:        "vz-correctness-fork",
+		FromRunning: true,
+		TargetState: StateRunning,
+	})
+	if err != nil {
+		dumpVZShimLogs(t, tmpDir)
+		require.NoError(t, err)
+	}
+	forkID := fork.Id
+	t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), mgr, forkID) })
+	require.NoError(t, waitForExecAgent(ctx, mgr, sourceID, 60*time.Second))
+	require.NoError(t, waitForExecAgent(ctx, mgr, forkID, 60*time.Second))
+	source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 30*time.Second)
+	require.NoError(t, err)
+	fork, err = waitForInstanceState(ctx, mgr, forkID, StateRunning, 30*time.Second)
+	require.NoError(t, err)
+	require.NotEqual(t, source.IP, fork.IP)
+	require.NotEqual(t, source.MAC, fork.MAC)
+	writeVZForkMarker(t, ctx, source, "source")
+	writeVZForkMarker(t, ctx, fork, "fork")
+	assertVZForkWorkload(t, ctx, source, fork, before)
+
+	for i := 0; i < 2; i++ {
+		beforeRestore := readVZForkWorkloadState(t, ctx, fork)
+		_, err = mgr.StandbyInstance(ctx, forkID, StandbyInstanceRequest{})
+		require.NoError(t, err)
+		_, err = mgr.RestoreInstance(ctx, forkID)
+		require.NoError(t, err)
+		require.NoError(t, waitForExecAgent(ctx, mgr, forkID, 60*time.Second))
+		fork, err = waitForInstanceState(ctx, mgr, forkID, StateRunning, 30*time.Second)
+		require.NoError(t, err)
+		afterRestore := readVZForkWorkloadState(t, ctx, fork)
+		require.GreaterOrEqual(t, afterRestore.counter, beforeRestore.counter)
+		assertVZForkWorkload(t, ctx, source, fork, before)
+	}
+
+	t.Run("host-http", func(t *testing.T) {
+		if os.Getenv("HYPEMAN_VZ_FORK_HTTP") != "1" {
+			t.Skip("set HYPEMAN_VZ_FORK_HTTP=1 to check forked NAT reachability")
+		}
+		assertVZForkHTTP(t, ctx, source, "source")
+		assertVZForkHTTP(t, ctx, fork, "fork")
+	})
+}
+
+// The shell counter and tmpfs nonce distinguish a RAM restore from a cold boot.
+const vzForkWorkload = `
+mkdir -p /dev/shm
+mount -t tmpfs -o size=16m tmpfs /dev/shm
+cat /proc/sys/kernel/random/uuid > /dev/shm/fork-nonce
+echo $$ > /dev/shm/fork-pid
+echo shared > /dev/shm/fork-marker
+echo shared > /usr/share/nginx/html/index.html
+nginx -g 'daemon off;' &
+i=0
+while :; do
+  i=$((i + 1))
+  echo "$i" > /dev/shm/fork-counter.tmp
+  mv /dev/shm/fork-counter.tmp /dev/shm/fork-counter
+  sleep 0.1
+done
+`
+
+type vzForkWorkloadState struct {
+	bootID  string
+	pid     string
+	nonce   string
+	counter uint64
+}
+
+func readVZForkWorkloadState(t *testing.T, ctx context.Context, inst *Instance) vzForkWorkloadState {
+	t.Helper()
+	out, code, err := vzExecCommand(ctx, inst, "sh", "-ec",
+		"cat /proc/sys/kernel/random/boot_id /dev/shm/fork-pid /dev/shm/fork-nonce /dev/shm/fork-counter")
+	require.NoError(t, err)
+	require.Equal(t, 0, code, out)
+	fields := strings.Fields(out)
+	require.Len(t, fields, 4)
+	counter, err := strconv.ParseUint(fields[3], 10, 64)
+	require.NoError(t, err)
+	return vzForkWorkloadState{bootID: fields[0], pid: fields[1], nonce: fields[2], counter: counter}
+}
+
+func assertVZForkWorkload(t *testing.T, ctx context.Context, source, fork *Instance, before vzForkWorkloadState) {
+	t.Helper()
+	for _, inst := range []*Instance{source, fork} {
+		state := readVZForkWorkloadState(t, ctx, inst)
+		require.Equal(t, before.bootID, state.bootID, "guest must not cold-boot")
+		require.Equal(t, before.pid, state.pid, "workload process must survive the fork")
+		require.Equal(t, before.nonce, state.nonce, "tmpfs contents must survive the fork")
+		require.GreaterOrEqual(t, state.counter, before.counter)
+		require.Eventually(t, func() bool {
+			out, code, err := vzExecCommand(ctx, inst, "cat", "/dev/shm/fork-counter")
+			counter, parseErr := strconv.ParseUint(strings.TrimSpace(out), 10, 64)
+			return err == nil && code == 0 && parseErr == nil && counter > state.counter
+		}, 10*time.Second, 100*time.Millisecond, "guest workload must continue making progress")
+	}
+
+	guests := []struct {
+		inst   *Instance
+		marker string
+	}{
+		{source, "source"},
+		{fork, "fork"},
+	}
+	for _, item := range guests {
+		out, code, err := vzExecCommand(ctx, item.inst, "sh", "-ec",
+			"cat /dev/shm/fork-marker /usr/share/nginx/html/index.html")
+		require.NoError(t, err)
+		require.Equal(t, 0, code)
+		require.Equal(t, []string{item.marker, item.marker}, strings.Fields(out), "guest RAM and disk writes must be independent")
+	}
+
+	// A smoke check for duplicated random streams, not a cryptographic audit.
+	sourceRandom, code, err := vzExecCommand(ctx, source, "cat", "/proc/sys/kernel/random/uuid")
+	require.NoError(t, err)
+	require.Equal(t, 0, code)
+	forkRandom, code, err := vzExecCommand(ctx, fork, "cat", "/proc/sys/kernel/random/uuid")
+	require.NoError(t, err)
+	require.Equal(t, 0, code)
+	require.NotEmpty(t, strings.TrimSpace(sourceRandom))
+	require.NotEmpty(t, strings.TrimSpace(forkRandom))
+	require.NotEqual(t, sourceRandom, forkRandom, "fresh random output should diverge after the fork")
+}
+
+func writeVZForkMarker(t *testing.T, ctx context.Context, inst *Instance, marker string) {
+	t.Helper()
+	_, code, err := vzExecCommand(ctx, inst, "sh", "-ec",
+		fmt.Sprintf("echo %s > /dev/shm/fork-marker; echo %s > /usr/share/nginx/html/index.html", marker, marker))
+	require.NoError(t, err)
+	require.Equal(t, 0, code)
+}
+
+func assertVZForkHTTP(t *testing.T, ctx context.Context, inst *Instance, marker string) {
+	t.Helper()
+	client := &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{}}
+	defer client.CloseIdleConnections()
+	url := "http://" + net.JoinHostPort(inst.IP, "80") + "/"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if t.Failed() {
+			diagCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			out, _, err := vzExecCommand(diagCtx, inst, "sh", "-c", "ip addr show eth0; ip route; wget -qO- http://127.0.0.1/")
+			t.Logf("guest HTTP diagnostics: %s (exec error: %v)", out, err)
+		}
+	})
+	require.Eventually(t, func() bool {
+		resp, err := client.Do(req)
+		if err != nil {
+			return false
+		}
+		defer resp.Body.Close()
+		body, err := io.ReadAll(io.LimitReader(resp.Body, 1024))
+		return err == nil && resp.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == marker
+	}, 20*time.Second, 200*time.Millisecond, "host HTTP request must reach the correct guest at %s", url)
+}