From 876bf41779755c978ecf186b9e4555ac2b280af1 Mon Sep 17 00:00:00 2001 From: Daniel Cazzulino Date: Mon, 26 Oct 2026 15:41:04 -0300 Subject: [PATCH] Wait for processes a tool started to exit before the update restarts it. A graceful close lets the root exit while its workers are reparented, so killing that pid no longer reaches them. --- readme.md | 3 +- src/Tests/CodeAltaEvergreenTests.cs | 694 ++++++++++++++++++++++ src/Tests/CodeAltaSourceEvergreenTests.cs | 492 +++++++++++++++ src/ndx/ChildProcess.cs | 138 ++++- src/ndx/Evergreen.cs | 1 + src/ndx/ProcessStop.cs | 1 + src/ndx/ProcessTree.cs | 194 ++++++ src/ndx/ndx.csproj | 1 + 8 files changed, 1512 insertions(+), 12 deletions(-) create mode 100644 src/Tests/CodeAltaEvergreenTests.cs create mode 100644 src/Tests/CodeAltaSourceEvergreenTests.cs create mode 100644 src/ndx/ProcessTree.cs diff --git a/readme.md b/readme.md index 283b77f..823dde0 100644 --- a/readme.md +++ b/readme.md @@ -136,7 +136,8 @@ Unlike `dnx`, omitting the version is `@*` — not a one-shot latest. `ndx my-agent` and `ndx my-agent@*` are the same: start the latest match, and while that process is still running, watch the feed, download a newer matching version *before* stopping the child, then send SIGINT / Ctrl+C -(and `WM_CLOSE` if the tool is a GUI) and restart. +(and `WM_CLOSE` if the tool is a GUI). The new process starts once that +child and the processes it started have exited. `@*-*` includes prereleases. `@1.*` / `@1.1.*` pin a major or minor the GitHub Actions way (float the rest). A NuGet range like `[1.0,2.0)` works diff --git a/src/Tests/CodeAltaEvergreenTests.cs b/src/Tests/CodeAltaEvergreenTests.cs new file mode 100644 index 0000000..925d3b4 --- /dev/null +++ b/src/Tests/CodeAltaEvergreenTests.cs @@ -0,0 +1,694 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using ndx; + +namespace Tests; + +public class CodeAltaEvergreenTests +{ + [Fact] + public async Task Simulated_update_sends_close_and_reaps_alta_processes_before_restart() + { + var published = CodeAltaSim.Publish(); + var root = Path.Combine(Path.GetTempPath(), "ndx-codealta-run", Guid.NewGuid().ToString("n")); + var feed = Path.Combine(root, "feed"); + var work = Path.Combine(root, "work"); + var store = Path.Combine(root, "store"); + Directory.CreateDirectory(feed); + Directory.CreateDirectory(work); + CodeAltaSim.Pack(published, feed, "1.0.0"); + + var output = new StringWriter(); + var error = new StringWriter(); + var host = new NdxHost + { + WorkingDirectory = work, + StoreDirectory = store, + ProcessRunner = new ProcessRunner(), + Out = output, + Error = error, + UpdateInterval = TimeSpan.FromMilliseconds(200), + StopTimeout = TimeSpan.FromSeconds(8), + }; + + var simDir = Path.Combine(work, "codealta-sim"); + var previous = Path.Combine(store, "codealta", "1.0.0"); + using var cts = new CancellationTokenSource(); + var run = App.RunAsync(["CodeAlta", "--yes", "--source", feed], host, cts.Token); + try + { + await WaitUntil( + () => File.Exists(Path.Combine(simDir, "ready")), + TimeSpan.FromSeconds(30), + () => $"CodeAlta did not become ready.{Environment.NewLine}{output}{Environment.NewLine}{error}{Environment.NewLine}{Read(Path.Combine(simDir, "error.txt"))}"); + + var workers = ReadWorkers(Path.Combine(simDir, "workers.txt")); + Assert.Contains(workers, worker => worker.Name.Equals("alta", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(workers, worker => worker.Name.Equals("CodeAlta", StringComparison.OrdinalIgnoreCase)); + Assert.All(workers, worker => Assert.True(IsAlive(worker.Pid), $"{worker.Name} {worker.Pid} was not alive before the update.")); + + var start = Read(Path.Combine(simDir, "start-1.txt")); + Assert.Contains("automated=1", start); + + CodeAltaSim.Pack(published, feed, "1.0.1"); + + await WaitUntil( + () => File.Exists(Path.Combine(simDir, "start-2.txt")), + TimeSpan.FromSeconds(20), + () => $"Updated CodeAlta did not start.{Environment.NewLine}{output}{Environment.NewLine}{error}{Environment.NewLine}{Read(Path.Combine(simDir, "error.txt"))}"); + + var signal = Read(Path.Combine(simDir, "signal.txt")).Trim(); + Assert.Equal(OperatingSystem.IsWindows() ? "WM_CLOSE" : "SIGINT", signal); + Assert.Contains("Updating CodeAlta 1.0.0 → 1.0.1", output.ToString()); + + var next = Read(Path.Combine(simDir, "start-2.txt")); + Assert.Contains("prior=", next); + var prior = next.Split('\n').Select(line => line.Trim('\r')).First(line => line.StartsWith("prior=", StringComparison.Ordinal)); + Assert.Equal("prior=", prior); + + var stillRunning = workers + .Where(worker => IsAltaUnder(worker.Pid, previous)) + .Select(worker => $"{worker.Name} {worker.Pid}") + .ToList(); + Assert.True(stillRunning.Count == 0, "Processes still alive after the close signal: " + string.Join(", ", stillRunning)); + + var leftovers = AltaUnder(previous); + Assert.True( + leftovers.Count == 0, + "alta/CodeAlta processes from 1.0.0 were still alive when 1.0.1 started: " + string.Join(", ", leftovers)); + } + finally + { + cts.Cancel(); + try + { + await run.WaitAsync(TimeSpan.FromSeconds(20)); + } + catch + { + } + + KillUnder(store); + } + } + + static List<(int Pid, string Name)> ReadWorkers(string path) + { + var workers = new List<(int Pid, string Name)>(); + foreach (var line in File.ReadAllLines(path)) + { + var split = line.Split(' ', 2, StringSplitOptions.RemoveEmptyEntries); + if (split.Length == 2 && int.TryParse(split[0], out var pid)) + workers.Add((pid, split[1])); + } + + return workers; + } + + static bool IsAlive(int pid) + { + try + { + using var process = Process.GetProcessById(pid); + return !process.HasExited; + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException) + { + return false; + } + } + + static bool IsAltaUnder(int pid, string directory) + { + try + { + using var process = Process.GetProcessById(pid); + if (process.HasExited) + return false; + + var name = process.ProcessName; + if (!name.Equals("alta", StringComparison.OrdinalIgnoreCase) && + !name.Equals("CodeAlta", StringComparison.OrdinalIgnoreCase)) + return false; + + var path = process.MainModule?.FileName; + return path is not null && IsUnder(path, directory); + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException or System.ComponentModel.Win32Exception) + { + return false; + } + } + + static List AltaUnder(string directory) + { + var found = new List(); + foreach (var process in Process.GetProcesses()) + { + try + { + var name = process.ProcessName; + if (!name.Equals("alta", StringComparison.OrdinalIgnoreCase) && + !name.Equals("CodeAlta", StringComparison.OrdinalIgnoreCase)) + continue; + + var path = process.MainModule?.FileName; + if (path is not null && IsUnder(path, directory)) + found.Add($"{process.Id} {name} {path}"); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + } + finally + { + process.Dispose(); + } + } + + return found; + } + + static void KillUnder(string directory) + { + foreach (var process in Process.GetProcesses()) + { + try + { + var name = process.ProcessName; + if (!name.Equals("alta", StringComparison.OrdinalIgnoreCase) && + !name.Equals("CodeAlta", StringComparison.OrdinalIgnoreCase)) + continue; + + var path = process.MainModule?.FileName; + if (path is null || !IsUnder(path, directory)) + continue; + + process.Kill(entireProcessTree: true); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception or ArgumentException) + { + } + finally + { + process.Dispose(); + } + } + } + + static bool IsUnder(string path, string directory) + { + var full = Path.GetFullPath(path); + var root = Path.GetFullPath(directory).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; + return full.StartsWith(root, StringComparison.OrdinalIgnoreCase); + } + + static string Read(string path) => File.Exists(path) ? File.ReadAllText(path) : ""; + + static async Task WaitUntil(Func condition, TimeSpan timeout, Func failure) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (condition()) + return; + await Task.Delay(20); + } + + throw new TimeoutException(failure()); + } +} + +static class CodeAltaSim +{ + const string SimSource = """ + using System.Diagnostics; + using System.Runtime.InteropServices; + + static class Program + { + static string simDirectory = ""; + static WndProcDel? windowProc; + + [STAThread] + static int Main(string[] args) + { + simDirectory = Path.Combine(Environment.CurrentDirectory, "codealta-sim"); + Directory.CreateDirectory(simDirectory); + + if (args.Contains("--worker")) + return Worker(); + + var prior = LivingRecorded(); + var self = Environment.ProcessPath ?? ""; + var folder = Path.GetDirectoryName(self) ?? ""; + var codeAlta = Path.Combine(folder, OperatingSystem.IsWindows() ? "CodeAlta.exe" : "CodeAlta"); + if (!File.Exists(codeAlta)) + { + File.WriteAllText(Path.Combine(simDirectory, "error.txt"), "Missing " + codeAlta); + return 1; + } + + if (!Start(self) || !Start(codeAlta)) + return 1; + + if (!WaitForWorkers(2, TimeSpan.FromSeconds(10))) + { + File.WriteAllText(Path.Combine(simDirectory, "error.txt"), "Workers did not record themselves."); + return 1; + } + + var generation = 1 + Directory.GetFiles(simDirectory, "start-*.txt").Length; + File.WriteAllText( + Path.Combine(simDirectory, $"start-{generation}.txt"), + $"pid={Environment.ProcessId}{Environment.NewLine}" + + $"prior={string.Join(',', prior)}{Environment.NewLine}" + + $"workers={string.Join(',', ReadWorkerIds())}{Environment.NewLine}" + + $"automated={Environment.GetEnvironmentVariable("CODEALTA_AUTOMATED")}{Environment.NewLine}"); + + return OperatingSystem.IsWindows() ? WindowsLoop() : UnixLoop(); + } + + static int Worker() + { + try + { + Console.CancelKeyPress += (_, e) => e.Cancel = true; + } + catch (InvalidOperationException) + { + } + + AppendLine(Path.Combine(simDirectory, "workers.txt"), Environment.ProcessId + " " + Process.GetCurrentProcess().ProcessName); + Thread.Sleep(Timeout.Infinite); + return 0; + } + + static bool Start(string file) + { + try + { + var started = Process.Start(new ProcessStartInfo(file, "--worker") + { + UseShellExecute = false, + WorkingDirectory = Environment.CurrentDirectory, + }); + if (started is not null) + return true; + + File.WriteAllText(Path.Combine(simDirectory, "error.txt"), "Failed to start " + file); + return false; + } + catch (Exception ex) + { + File.WriteAllText(Path.Combine(simDirectory, "error.txt"), ex.ToString()); + return false; + } + } + + static bool WaitForWorkers(int count, TimeSpan timeout) + { + var path = Path.Combine(simDirectory, "workers.txt"); + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + var lines = 0; + foreach (var line in ReadLines(path)) + { + if (line.Length > 0) + lines++; + } + + if (lines >= count) + return true; + + Thread.Sleep(20); + } + + return false; + } + + static List ReadWorkerIds() + { + var ids = new List(); + foreach (var line in ReadLines(Path.Combine(simDirectory, "workers.txt"))) + { + var split = line.Split(' ', 2); + if (int.TryParse(split[0], out var pid)) + ids.Add(pid); + } + + return ids; + } + + static string[] ReadLines(string path) + { + if (!File.Exists(path)) + return []; + + try + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var reader = new StreamReader(stream); + var lines = new List(); + while (reader.ReadLine() is { } line) + lines.Add(line); + return lines.ToArray(); + } + catch (IOException) + { + return []; + } + } + + static List LivingRecorded() + { + var ids = new HashSet(); + foreach (var file in Directory.GetFiles(simDirectory, "start-*.txt")) + { + foreach (var line in ReadLines(file)) + { + if (line.StartsWith("pid=", StringComparison.Ordinal) && int.TryParse(line.AsSpan(4), out var pid)) + ids.Add(pid); + if (!line.StartsWith("workers=", StringComparison.Ordinal)) + continue; + + foreach (var part in line["workers=".Length..].Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (int.TryParse(part, out var worker)) + ids.Add(worker); + } + } + } + + foreach (var pid in ReadWorkerIds()) + ids.Add(pid); + + var alive = new List(); + foreach (var pid in ids) + { + if (pid == Environment.ProcessId || !IsAlta(pid)) + continue; + alive.Add(pid); + } + + return alive; + } + + static bool IsAlta(int pid) + { + try + { + using var process = Process.GetProcessById(pid); + if (process.HasExited) + return false; + var name = process.ProcessName; + return name.Equals("alta", StringComparison.OrdinalIgnoreCase) || + name.Equals("CodeAlta", StringComparison.OrdinalIgnoreCase); + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException) + { + return false; + } + } + + static void AppendLine(string path, string line) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(line + "\n"); + for (var attempt = 0; ; attempt++) + { + try + { + using var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite); + stream.Write(bytes); + return; + } + catch (IOException) when (attempt < 50) + { + Thread.Sleep(10); + } + } + } + + static int UnixLoop() + { + var done = new ManualResetEventSlim(false); + Console.CancelKeyPress += (_, e) => + { + e.Cancel = true; + File.WriteAllText(Path.Combine(simDirectory, "signal.txt"), "SIGINT"); + done.Set(); + }; + File.WriteAllText(Path.Combine(simDirectory, "ready"), "1"); + done.Wait(); + return 0; + } + + static int WindowsLoop() + { + windowProc = WndProc; + var instance = GetModuleHandle(null); + var wndClass = new WndClass + { + lpfnWndProc = windowProc, + hInstance = instance, + lpszClassName = "NeoAstra.Window", + }; + var atom = RegisterClass(ref wndClass); + if (atom == 0) + { + File.WriteAllText(Path.Combine(simDirectory, "error.txt"), "RegisterClass failed: " + Marshal.GetLastWin32Error()); + return 1; + } + + var hwnd = CreateWindowEx(0, (nint)atom, "CodeAlta", 0x00CF0000, 0, 0, 80, 80, 0, 0, instance, 0); + if (hwnd == 0) + { + File.WriteAllText(Path.Combine(simDirectory, "error.txt"), "CreateWindowEx failed: " + Marshal.GetLastWin32Error()); + return 1; + } + + ShowWindow(hwnd, 0); + File.WriteAllText(Path.Combine(simDirectory, "ready"), "1"); + while (GetMessage(out var msg, 0, 0, 0) > 0) + { + TranslateMessage(ref msg); + DispatchMessage(ref msg); + } + + return 0; + } + + static nint WndProc(nint hWnd, uint msg, nint wParam, nint lParam) + { + if (msg == 0x0010) + { + File.WriteAllText(Path.Combine(simDirectory, "signal.txt"), "WM_CLOSE"); + PostQuitMessage(0); + return 0; + } + + return DefWindowProc(hWnd, msg, wParam, lParam); + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + struct WndClass + { + public uint style; + public WndProcDel lpfnWndProc; + public int cbClsExtra; + public int cbWndExtra; + public nint hInstance; + public nint hIcon; + public nint hCursor; + public nint hbrBackground; + public string? lpszMenuName; + public string lpszClassName; + } + + delegate nint WndProcDel(nint hWnd, uint msg, nint wParam, nint lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] + static extern nint GetModuleHandle(string? lpModuleName); + + [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + static extern ushort RegisterClass(ref WndClass lpWndClass); + + [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + static extern nint CreateWindowEx(uint dwExStyle, nint lpClassName, string lpWindowName, uint dwStyle, int x, int y, int nWidth, int nHeight, nint hWndParent, nint hMenu, nint hInstance, nint lpParam); + + [DllImport("user32.dll")] + static extern bool ShowWindow(nint hWnd, int nCmdShow); + + [DllImport("user32.dll")] + static extern int GetMessage(out Msg lpMsg, nint hWnd, uint wMsgFilterMin, uint wMsgFilterMax); + + [DllImport("user32.dll")] + static extern bool TranslateMessage(ref Msg lpMsg); + + [DllImport("user32.dll")] + static extern nint DispatchMessage(ref Msg lpMsg); + + [DllImport("user32.dll")] + static extern nint DefWindowProc(nint hWnd, uint msg, nint wParam, nint lParam); + + [DllImport("user32.dll")] + static extern void PostQuitMessage(int nExitCode); + + struct Msg + { + public nint hwnd; + public uint message; + public nint wParam; + public nint lParam; + public uint time; + public int ptX; + public int ptY; + } + } + """; + + static readonly object Gate = new(); + + public static string Publish() + { + lock (Gate) + { + var root = Path.Combine(Path.GetTempPath(), "ndx-codealta-sim"); + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(SimSource))); + var stamp = Path.Combine(root, "stamp.txt"); + var altaOut = Path.Combine(root, "publish-alta"); + var exeName = OperatingSystem.IsWindows() ? "alta.exe" : "alta"; + var codeName = OperatingSystem.IsWindows() ? "CodeAlta.exe" : "CodeAlta"; + if (File.Exists(stamp) && File.ReadAllText(stamp) == hash && + File.Exists(Path.Combine(altaOut, exeName)) && + File.Exists(Path.Combine(altaOut, codeName))) + return altaOut; + + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + + Directory.CreateDirectory(root); + var altaProj = Path.Combine(root, "alta"); + var codeProj = Path.Combine(root, "CodeAlta"); + Directory.CreateDirectory(altaProj); + Directory.CreateDirectory(codeProj); + File.WriteAllText(Path.Combine(altaProj, "Program.cs"), SimSource); + File.WriteAllText(Path.Combine(altaProj, "alta.csproj"), Project(includeProgram: false)); + File.WriteAllText(Path.Combine(codeProj, "CodeAlta.csproj"), Project(includeProgram: true)); + + var codeOut = Path.Combine(root, "publish-codealta"); + RunDotnet(altaProj, altaOut); + RunDotnet(codeProj, codeOut); + foreach (var file in Directory.GetFiles(codeOut)) + { + var name = Path.GetFileName(file); + if (name.StartsWith("CodeAlta", StringComparison.OrdinalIgnoreCase)) + File.Copy(file, Path.Combine(altaOut, name), overwrite: true); + } + + if (!File.Exists(Path.Combine(altaOut, codeName))) + throw new InvalidOperationException("CodeAlta apphost was not produced."); + + File.WriteAllText(stamp, hash); + return altaOut; + } + } + + public static void Pack(string publishDir, string feedDir, string version) + { + var staging = Path.Combine(Path.GetTempPath(), "ndx-codealta-nupkg", Guid.NewGuid().ToString("n")); + var tools = Path.Combine(staging, "tools", "net10.0", "any"); + Directory.CreateDirectory(tools); + foreach (var file in Directory.GetFiles(publishDir)) + File.Copy(file, Path.Combine(tools, Path.GetFileName(file)), overwrite: true); + + var exe = OperatingSystem.IsWindows() ? "alta.exe" : "alta"; + File.WriteAllText(Path.Combine(tools, "DotnetToolSettings.xml"), + $""" + + + + + + + """); + + File.WriteAllText(Path.Combine(staging, "CodeAlta.nuspec"), + $""" + + + + CodeAlta + {version} + ndx + simulated CodeAlta + + + + + + """); + + Directory.CreateDirectory(feedDir); + var nupkg = Path.Combine(feedDir, $"CodeAlta.{version}.nupkg"); + if (File.Exists(nupkg)) + File.Delete(nupkg); + ZipFile.CreateFromDirectory(staging, nupkg); + } + + static string Project(bool includeProgram) => + includeProgram + ? """ + + + WinExe + net10.0 + enable + enable + CodeAlta + true + false + + + + + + """ + : """ + + + WinExe + net10.0 + enable + enable + alta + true + + + """; + + static void RunDotnet(string projectDir, string output) + { + var start = new ProcessStartInfo(OperatingSystem.IsWindows() ? "dotnet.exe" : "dotnet") + { + WorkingDirectory = projectDir, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + start.ArgumentList.Add("publish"); + start.ArgumentList.Add("-c"); + start.ArgumentList.Add("Release"); + start.ArgumentList.Add("-o"); + start.ArgumentList.Add(output); + start.ArgumentList.Add("--nologo"); + var (exit, stdout, stderr) = ProcessCapture.Run(start, timeoutMs: 180_000); + if (exit != 0) + { + throw new InvalidOperationException( + $"dotnet publish failed ({exit}).{Environment.NewLine}{stdout}{Environment.NewLine}{stderr}"); + } + } +} diff --git a/src/Tests/CodeAltaSourceEvergreenTests.cs b/src/Tests/CodeAltaSourceEvergreenTests.cs new file mode 100644 index 0000000..4bfbc83 --- /dev/null +++ b/src/Tests/CodeAltaSourceEvergreenTests.cs @@ -0,0 +1,492 @@ +using System.Diagnostics; +using System.IO.Compression; +using ndx; + +namespace Tests; + +/// +/// Opt-in end to end against a Release build of the CodeAlta source +/// (NDX_CODEALTA_E2E=1). ndx posts the same close as a simulated update. +/// CodeAlta, started with CODEALTA_AUTOMATED=1, has to leave by its own +/// automated exit: exit code 0, the desktop log says so, and nothing that +/// outlived that process was terminated. +/// +public class CodeAltaSourceEvergreenTests +{ + [Fact(Timeout = 1_200_000)] + public async Task Owned_desktop_update_closes_through_automated_exit() + { + if (Environment.GetEnvironmentVariable("NDX_CODEALTA_E2E") != "1") + return; + + var repo = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..")); + var project = Path.GetFullPath(Path.Combine(repo, "..", "oss", "CodeAlta", "src", "CodeAlta", "CodeAlta.csproj")); + Assert.True(File.Exists(project), "CodeAlta.csproj was not found at " + project); + + var built = BuiltOutput(project); + var root = Path.Combine(Path.GetTempPath(), "ndx-codealta-source", Guid.NewGuid().ToString("n")); + var feed = Path.Combine(root, "feed"); + var work = Path.Combine(root, "work"); + var store = Path.Combine(root, "store"); + var data = Path.Combine(root, "data"); + var closedData = data + "-closed"; + var catalog = Path.Combine(root, "catalog"); + var projectRoot = Path.Combine(root, "project"); + var home = Path.Combine(root, "home"); + var skills = Path.Combine(root, "skills"); + Directory.CreateDirectory(work); + Directory.CreateDirectory(catalog); + Directory.CreateDirectory(projectRoot); + Directory.CreateDirectory(home); + Directory.CreateDirectory(skills); + + Pack(built, feed, "1.0.0"); + var nextPackage = Path.Combine(root, "next.nupkg"); + var packingNext = Task.Run(() => Pack(built, nextPackage, "1.0.1")); + + var output = new StringWriter(); + var error = new StringWriter(); + var runner = new FreshDataRootRunner(new ProcessRunner(), data); + var host = new NdxHost + { + WorkingDirectory = work, + StoreDirectory = store, + ProcessRunner = runner, + Out = output, + Error = error, + UpdateInterval = TimeSpan.FromMilliseconds(500), + StopTimeout = TimeSpan.FromSeconds(90), + }; + + var versionDir = Path.Combine(store, "codealta", "1.0.0"); + using var cts = new CancellationTokenSource(); + var run = App.RunAsync( + [ + "CodeAlta", "--yes", "--source", feed, "--", + "--data-root", data, + "--catalog-root", catalog, + "--allow-catalog-cache", + "--allow-owned-host", + "--project-root", projectRoot, + "--discovery-home", home, + "--instruction-root", projectRoot, + "--builtin-skill-root", skills, + "--mcp-port", "0", + ], + host, + cts.Token); + + Process? main = null; + var held = new List(); + try + { + await WaitUntil( + () => LogHas(data, "Startup: application loading"), + TimeSpan.FromMinutes(3), + () => $"CodeAlta did not finish starting.{Environment.NewLine}{output}{error}{Environment.NewLine}{ReadLogs(data)}"); + + main = FindAlta(versionDir); + Assert.True(main is not null, "alta.exe from 1.0.0 was not running after startup."); + var tracked = new HashSet(); + ProcessTree.AddDescendants(main.Id, tracked); + foreach (var pid in tracked) + { + try + { + held.Add(Process.GetProcessById(pid)); + } + catch (ArgumentException) + { + } + } + + await packingNext; + Directory.CreateDirectory(feed); + var stop = System.Diagnostics.Stopwatch.StartNew(); + File.Move(nextPackage, Path.Combine(feed, "CodeAlta.1.0.1.nupkg"), overwrite: true); + + var aliveAfterParent = new List(); + var sawParentExit = false; + await WaitUntil( + () => + { + if (!sawParentExit && main.HasExited) + { + sawParentExit = true; + foreach (var child in held) + { + try + { + child.Refresh(); + if (!child.HasExited) + aliveAfterParent.Add(child); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + } + } + } + + return runner.FirstExit.IsCompleted; + }, + TimeSpan.FromMinutes(2), + () => $"CodeAlta 1.0.0 did not exit after the update.{Environment.NewLine}{output}{error}{Environment.NewLine}{ReadLogs(Directory.Exists(closedData) ? closedData : data)}"); + + var exitCode = await runner.FirstExit.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, exitCode); + Assert.True(stop.Elapsed < host.StopTimeout, $"CodeAlta was still up after {stop.Elapsed}, the point where ndx kills the process."); + Assert.Contains("Updating CodeAlta 1.0.0 → 1.0.1", output.ToString()); + + var closedLog = ReadLogs(closedData); + Assert.Contains("Closing the window (User): ExitWhenIdle.", closedLog); + Assert.Contains("The application exits: its host closes.", closedLog); + Assert.Contains("The host is closed.", closedLog); + + foreach (var child in aliveAfterParent) + { + child.Refresh(); + // ExitCode is only available for a process this test started. The child exiting + // before the stop timeout is the graceful case: ndx kills a descendant only after that wait. + Assert.True(child.HasExited, Describe(child) + " was still alive after CodeAlta exited."); + } + + var leftovers = AltaUnder(versionDir); + Assert.True(leftovers.Count == 0, "1.0.0 processes still alive when the update started the next process: " + string.Join(", ", leftovers)); + + await WaitUntil( + () => LogHas(data, "Startup: application loading"), + TimeSpan.FromMinutes(3), + () => $"Updated CodeAlta did not start.{Environment.NewLine}{output}{error}{Environment.NewLine}{ReadLogs(data)}"); + + cts.Cancel(); + var restarted = await run.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.Equal(0, restarted); + await WaitUntil( + () => LogHas(data, "The host is closed."), + TimeSpan.FromSeconds(20), + () => $"The restarted CodeAlta did not record a graceful exit.{Environment.NewLine}{ReadLogs(data)}"); + Assert.Contains("Closing the window (User): ExitWhenIdle.", ReadLogs(data)); + } + finally + { + cts.Cancel(); + try + { + await run.WaitAsync(TimeSpan.FromSeconds(30)); + } + catch + { + } + + main?.Dispose(); + foreach (var child in held) + child.Dispose(); + KillUnder(store); + try + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + catch (IOException) + { + } + } + } + + static string BuiltOutput(string project) + { + var output = Path.Combine(Path.GetDirectoryName(project)!, "bin", "Release", "net10.0"); + var exe = Path.Combine(output, OperatingSystem.IsWindows() ? "alta.exe" : "alta"); + Assert.True(File.Exists(exe), "Release build of CodeAlta was not found at " + exe); + return output; + } + + static void Pack(string publishDir, string destination, string version) + { + var staging = Path.Combine(Path.GetTempPath(), "ndx-codealta-nupkg", Guid.NewGuid().ToString("n")); + var tools = Path.Combine(staging, "tools", "net10.0", "any"); + CopyTree(publishDir, tools); + var exe = OperatingSystem.IsWindows() ? "alta.exe" : "alta"; + File.WriteAllText(Path.Combine(tools, "DotnetToolSettings.xml"), + $""" + + + + + + + """); + File.WriteAllText(Path.Combine(staging, "CodeAlta.nuspec"), + $""" + + + + CodeAlta + {version} + ndx + CodeAlta built from source + + + + + + """); + + var nupkg = destination.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) + ? destination + : Path.Combine(destination, $"CodeAlta.{version}.nupkg"); + Directory.CreateDirectory(Path.GetDirectoryName(nupkg)!); + if (File.Exists(nupkg)) + File.Delete(nupkg); + ZipFile.CreateFromDirectory(staging, nupkg); + Directory.Delete(staging, recursive: true); + } + + static readonly HashSet RidOutputs = new(StringComparer.OrdinalIgnoreCase) + { + "win-x64", "win-arm64", "linux-x64", "linux-arm64", "osx-x64", "osx-arm64", "ref", + }; + + static void CopyTree(string source, string destination) + { + Directory.CreateDirectory(destination); + foreach (var file in Directory.GetFiles(source)) + { + if (file.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase)) + continue; + File.Copy(file, Path.Combine(destination, Path.GetFileName(file)), overwrite: true); + } + + foreach (var directory in Directory.GetDirectories(source)) + { + var name = Path.GetFileName(directory); + if (RidOutputs.Contains(name)) + continue; + + if (name.Equals("runtimes", StringComparison.OrdinalIgnoreCase)) + { + var runtimes = Path.Combine(destination, name); + Directory.CreateDirectory(runtimes); + foreach (var rid in new[] { "win-x64", "win" }) + { + var from = Path.Combine(directory, rid); + if (Directory.Exists(from)) + CopyAll(from, Path.Combine(runtimes, rid)); + } + + continue; + } + + CopyTree(directory, Path.Combine(destination, name)); + } + } + + static void CopyAll(string source, string destination) + { + Directory.CreateDirectory(destination); + foreach (var file in Directory.GetFiles(source)) + { + if (file.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase)) + continue; + File.Copy(file, Path.Combine(destination, Path.GetFileName(file)), overwrite: true); + } + + foreach (var directory in Directory.GetDirectories(source)) + CopyAll(directory, Path.Combine(destination, Path.GetFileName(directory))); + } + + static Process? FindAlta(string directory) + { + foreach (var process in Process.GetProcessesByName("alta")) + { + try + { + var path = process.MainModule?.FileName; + if (path is not null && IsUnder(path, directory)) + return process; + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception) + { + } + + process.Dispose(); + } + + return null; + } + + static List AltaUnder(string directory) + { + var found = new List(); + foreach (var process in Process.GetProcessesByName("alta")) + { + try + { + if (process.HasExited) + continue; + var path = process.MainModule?.FileName; + if (path is not null && IsUnder(path, directory)) + found.Add($"{process.Id} {path}"); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception or ArgumentException) + { + } + finally + { + process.Dispose(); + } + } + + return found; + } + + static void KillUnder(string directory) + { + foreach (var process in Process.GetProcessesByName("alta")) + { + try + { + var path = process.MainModule?.FileName; + if (path is null || !IsUnder(path, directory)) + continue; + process.Kill(entireProcessTree: true); + } + catch (Exception ex) when (ex is InvalidOperationException or System.ComponentModel.Win32Exception or ArgumentException) + { + } + finally + { + process.Dispose(); + } + } + } + + static bool IsUnder(string path, string directory) + { + var full = Path.GetFullPath(path); + var root = Path.GetFullPath(directory).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; + return full.StartsWith(root, StringComparison.OrdinalIgnoreCase); + } + + static bool LogHas(string dataRoot, string text) + { + var logs = Path.Combine(dataRoot, "logs"); + if (!Directory.Exists(logs)) + return false; + + foreach (var file in Directory.GetFiles(logs)) + { + if (ReadShared(file).Contains(text, StringComparison.Ordinal)) + return true; + } + + return false; + } + + static string ReadLogs(string dataRoot) + { + var logs = Path.Combine(dataRoot, "logs"); + if (!Directory.Exists(logs)) + return ""; + + return string.Join(Environment.NewLine, Directory.GetFiles(logs).Select(file => Path.GetFileName(file) + Environment.NewLine + ReadShared(file))); + } + + static string ReadShared(string path) + { + try + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + catch (IOException) + { + return ""; + } + } + + static string Describe(Process process) + { + try + { + return $"{process.Id} {process.ProcessName}"; + } + catch (InvalidOperationException) + { + return process.Id.ToString(); + } + } + + static async Task WaitUntil(Func condition, TimeSpan timeout, Func failure) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (condition()) + return; + await Task.Delay(50); + } + + throw new TimeoutException(failure()); + } + + sealed class FreshDataRootRunner(IProcessRunner inner, string dataRoot) : IProcessRunner + { + readonly TaskCompletionSource firstExit = new(TaskCreationOptions.RunContinuationsAsynchronously); + int starts; + + public Task FirstExit => firstExit.Task; + + public int Run(ProcessStartSettings settings) => inner.Run(settings); + + public IChildProcess Start(ProcessStartSettings settings) + { + var generation = Interlocked.Increment(ref starts); + if (generation > 1) + MoveAside(dataRoot); + + var child = inner.Start(settings); + return generation == 1 ? new RememberExit(child, firstExit) : child; + } + + static void MoveAside(string source) + { + var destination = source + "-closed"; + for (var attempt = 0; ; attempt++) + { + try + { + if (Directory.Exists(destination)) + Directory.Delete(destination, recursive: true); + if (Directory.Exists(source)) + Directory.Move(source, destination); + return; + } + catch (IOException) when (attempt < 50) + { + Thread.Sleep(100); + } + } + } + } + + sealed class RememberExit(IChildProcess inner, TaskCompletionSource exited) : IChildProcess + { + public int Id => inner.Id; + public bool HasExited => inner.HasExited; + public int ExitCode => inner.ExitCode; + + public Task WaitForExitAsync(CancellationToken cancellationToken = default) + => inner.WaitForExitAsync(cancellationToken); + + public async Task StopAsync(TimeSpan timeout, CancellationToken cancellationToken = default) + { + await inner.StopAsync(timeout, cancellationToken).ConfigureAwait(false); + if (inner.HasExited) + exited.TrySetResult(inner.ExitCode); + } + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } +} diff --git a/src/ndx/ChildProcess.cs b/src/ndx/ChildProcess.cs index 9ee9e8d..4f1372a 100644 --- a/src/ndx/ChildProcess.cs +++ b/src/ndx/ChildProcess.cs @@ -1,3 +1,4 @@ +using System.ComponentModel; using System.Diagnostics; namespace ndx; @@ -48,21 +49,136 @@ async Task WaitCoreAsync(CancellationToken cancellationToken) public async Task StopAsync(TimeSpan timeout, CancellationToken cancellationToken = default) { - if (HasExited) - return; + // Remember descendants before the stop signal. After this process exits they are + // reparented, and a later kill of this pid no longer reaches them. + var tracked = new HashSet(); + if (!HasExited) + { + ProcessTree.AddDescendants(process.Id, tracked); + ProcessStop.Signal(process); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(timeout); + using var refresh = new CancellationTokenSource(); + var refreshing = RefreshDescendantsAsync(tracked, refresh.Token); + try + { + await process.WaitForExitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + if (!HasExited) + { + try + { + process.Kill(entireProcessTree: true); + } + catch (Exception ex) when (ex is InvalidOperationException or Win32Exception) + { + } + } + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + refresh.Cancel(); + try + { + await refreshing.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + } + } - ProcessStop.Signal(process); - using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - cts.CancelAfter(timeout); - try + ProcessTree.AddDescendants(process.Id, tracked); + // Descendants of a graceful exit are already stopping. Wait for that before killing anyone. + await ReapAsync(tracked, timeout).ConfigureAwait(false); + } + + async Task RefreshDescendantsAsync(HashSet tracked, CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested && !HasExited) { - await process.WaitForExitAsync(cts.Token).ConfigureAwait(false); + ProcessTree.AddDescendants(process.Id, tracked); + try + { + await Task.Delay(25, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } } - catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + + ProcessTree.AddDescendants(process.Id, tracked); + } + + async Task ReapAsync(HashSet pids, TimeSpan timeout) + { + var children = new List<(int Pid, Process Process)>(); + foreach (var pid in pids) { - if (!HasExited) - process.Kill(entireProcessTree: true); - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + if (pid <= 4 || pid == Environment.ProcessId || pid == process.Id) + continue; + + try + { + var child = Process.GetProcessById(pid); + if (child.HasExited) + { + child.Dispose(); + continue; + } + + children.Add((pid, child)); + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException or Win32Exception) + { + } + } + + await Task.WhenAll(children.Select(item => WaitThenKillAsync(item.Pid, item.Process, timeout))).ConfigureAwait(false); + } + + static async Task WaitThenKillAsync(int pid, Process child, TimeSpan timeout) + { + using (child) + { + using var wait = new CancellationTokenSource(timeout); + try + { + if (!child.HasExited) + await child.WaitForExitAsync(wait.Token).ConfigureAwait(false); + return; + } + catch (OperationCanceledException) + { + } + + if (child.HasExited) + return; + + try + { + child.Kill(entireProcessTree: true); + } + catch (Exception ex) when (ex is InvalidOperationException or Win32Exception) + { + } + + using var killed = new CancellationTokenSource(timeout); + try + { + if (!child.HasExited) + await child.WaitForExitAsync(killed.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + if (!child.HasExited) + throw new InvalidOperationException($"Process {pid} did not exit."); + } } } diff --git a/src/ndx/Evergreen.cs b/src/ndx/Evergreen.cs index 881e89f..32f79a1 100644 --- a/src/ndx/Evergreen.cs +++ b/src/ndx/Evergreen.cs @@ -74,6 +74,7 @@ public static async Task RunAsync( if (!quiet) host.Out.WriteLine($"Updating {invocation.PackageId} {command.Version} → {update.Version}"); + // StopAsync returns once this child and the processes it started have exited. using (shutdown.StoppingChild()) await child.StopAsync(host.StopTimeout, CancellationToken.None).ConfigureAwait(false); diff --git a/src/ndx/ProcessStop.cs b/src/ndx/ProcessStop.cs index 1ffe404..28da1b5 100644 --- a/src/ndx/ProcessStop.cs +++ b/src/ndx/ProcessStop.cs @@ -8,6 +8,7 @@ namespace ndx; /// /// Gracefully stops a child we started: SIGINT / Ctrl+C, then Windows /// WM_CLOSE if the tool is a GUI, then after a timeout. +/// Processes that child started are stopped as well, before the caller starts another one. /// A NeoAstra app is closed through its unowned top-level NeoAstra.Window frames only. /// Any other GUI keeps the generic close. /// In-process GenerateConsoleCtrlEvent is enough — the parent survives when diff --git a/src/ndx/ProcessTree.cs b/src/ndx/ProcessTree.cs new file mode 100644 index 0000000..afaf442 --- /dev/null +++ b/src/ndx/ProcessTree.cs @@ -0,0 +1,194 @@ +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace ndx; + +/// +/// Child processes of a process we started. After the parent exits, the OS +/// reparents them, so their ids have to be recorded while the parent is alive. +/// +static class ProcessTree +{ + public static void AddDescendants(int root, HashSet into) + { + var children = ChildrenByParent(); + var queue = new Queue(); + queue.Enqueue(root); + foreach (var pid in into) + queue.Enqueue(pid); + + var seen = new HashSet(); + while (queue.Count > 0) + { + var pid = queue.Dequeue(); + if (!seen.Add(pid)) + continue; + + if (pid != root) + into.Add(pid); + + if (!children.TryGetValue(pid, out var kids)) + continue; + + foreach (var kid in kids) + queue.Enqueue(kid); + } + } + + static Dictionary> ChildrenByParent() + { + var children = new Dictionary>(); + foreach (var (pid, ppid) in ReadParentMap()) + { + if (!children.TryGetValue(ppid, out var list)) + { + list = []; + children[ppid] = list; + } + + list.Add(pid); + } + + return children; + } + + static Dictionary ReadParentMap() + { + if (OperatingSystem.IsWindows()) + return ReadWindowsParents(); + + if (OperatingSystem.IsLinux()) + { + var proc = ReadProcParents(); + if (proc.Count > 0) + return proc; + } + + return ReadPsParents(); + } + + static Dictionary ReadProcParents() + { + var map = new Dictionary(); + if (!Directory.Exists("/proc")) + return map; + + foreach (var dir in Directory.EnumerateDirectories("/proc")) + { + var name = Path.GetFileName(dir); + if (!int.TryParse(name, out var pid)) + continue; + + try + { + var stat = File.ReadAllText(Path.Combine(dir, "stat")); + var end = stat.LastIndexOf(')'); + if (end < 0 || end + 2 >= stat.Length) + continue; + + var fields = stat[(end + 2)..].Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (fields.Length >= 2 && int.TryParse(fields[1], out var ppid)) + map[pid] = ppid; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + } + } + + return map; + } + + static Dictionary ReadPsParents() + { + var map = new Dictionary(); + try + { + using var ps = Process.Start(new ProcessStartInfo("ps", "-ax -o pid=,ppid=") + { + UseShellExecute = false, + RedirectStandardOutput = true, + CreateNoWindow = true, + }); + if (ps is null) + return map; + + var text = ps.StandardOutput.ReadToEnd(); + ps.WaitForExit(); + foreach (var line in text.Split('\n')) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && + int.TryParse(parts[0], out var pid) && + int.TryParse(parts[1], out var ppid)) + map[pid] = ppid; + } + } + catch (Exception ex) when (ex is InvalidOperationException or Win32Exception or IOException) + { + } + + return map; + } + +#if NDX_WINDOWS + const uint Th32CsSnapProcess = 2; + + static Dictionary ReadWindowsParents() + { + var map = new Dictionary(); + var snapshot = CreateToolhelp32Snapshot(Th32CsSnapProcess, 0); + if (snapshot == nint.Zero || snapshot == new nint(-1)) + return map; + + try + { + var entry = new ProcessEntry { dwSize = (uint)Marshal.SizeOf() }; + if (!Process32First(snapshot, ref entry)) + return map; + + do + { + map[(int)entry.th32ProcessID] = (int)entry.th32ParentProcessID; + } + while (Process32Next(snapshot, ref entry)); + + return map; + } + finally + { + CloseHandle(snapshot); + } + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + struct ProcessEntry + { + public uint dwSize; + public uint cntUsage; + public uint th32ProcessID; + public nint th32DefaultHeapID; + public uint th32ModuleID; + public uint cntThreads; + public uint th32ParentProcessID; + public int pcPriClassBase; + public uint dwFlags; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 260)] + public string szExeFile; + } + + [DllImport("kernel32.dll", SetLastError = true)] + static extern nint CreateToolhelp32Snapshot(uint dwFlags, uint th32ProcessID); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true, EntryPoint = "Process32FirstW")] + static extern bool Process32First(nint hSnapshot, ref ProcessEntry lppe); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true, EntryPoint = "Process32NextW")] + static extern bool Process32Next(nint hSnapshot, ref ProcessEntry lppe); + + [DllImport("kernel32.dll", SetLastError = true)] + static extern bool CloseHandle(nint hObject); +#else + static Dictionary ReadWindowsParents() => []; +#endif +} diff --git a/src/ndx/ndx.csproj b/src/ndx/ndx.csproj index 25179c7..f13c19a 100644 --- a/src/ndx/ndx.csproj +++ b/src/ndx/ndx.csproj @@ -27,6 +27,7 @@ +