Repository navigation
Relaunch the gateway on a restart request #4
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| use super::*; | ||
|
|
||
| #[test] | ||
| fn the_restart_backoff_starts_at_250_ms_and_doubles() { | ||
| let waits: Vec<_> = (1..MAX_RESTART_LAUNCHES).map(backoff).collect(); | ||
|
|
||
| assert_eq!( | ||
| waits, | ||
| [250, 500, 1000, 2000].map(Duration::from_millis), | ||
| "a restart waits before each of its launches after the first" | ||
| ); | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,51 +7,224 @@ | |
| //! owns the log file. | ||
|
|
||
| use std::io::{BufRead, BufReader, Read, Write}; | ||
| use std::process::{Command, ExitCode, ExitStatus, Stdio}; | ||
| use std::path::PathBuf; | ||
| use std::process::{Child, ChildStdin, Command, ExitCode, ExitStatus, Stdio}; | ||
| use std::thread::JoinHandle; | ||
| use std::time::{Duration, Instant}; | ||
|
|
||
| use gateway::AppOptions; | ||
| use gateway_api_discovery::LaunchLock; | ||
| use gateway::{AppOptions, RESTART_REQUESTED}; | ||
| use gateway_api_discovery::{GatewayDiscoveryFile, LaunchLock}; | ||
|
|
||
| /// How long a launched child has to publish its discovery file. | ||
| const PUBLICATION_TIMEOUT: Duration = Duration::from_secs(30); | ||
|
|
||
| /// How often a launching child is checked for publication or exit. | ||
| const PUBLICATION_POLL: Duration = Duration::from_millis(25); | ||
|
|
||
| /// The launches one restart may make before the supervisor gives up. | ||
| const MAX_RESTART_LAUNCHES: u32 = 5; | ||
|
|
||
| /// The wait before a restart's second launch; it doubles for each launch after. | ||
| const BASE_BACKOFF_TIMEOUT: Duration = Duration::from_millis(250); | ||
|
|
||
| /// Supervisor entry point: starts and manages the child gateway process, | ||
| /// returning the process exit code when it ends. Holds the launch lock while | ||
| /// the gateway process is running, so another launch attempt attaches to the | ||
| /// live child instead of launching a new gateway. | ||
| pub(crate) fn run(options: &AppOptions, _launch_lock: LaunchLock) -> ExitCode { | ||
| let mut child = match build_child_command(options).spawn() { | ||
| Ok(child) => child, | ||
| Err(error) => { | ||
| eprintln!("error: could not start the gateway: {error}"); | ||
| return ExitCode::FAILURE; | ||
| } | ||
| let Some(run_dir) = options.serve.resolved_run_dir() else { | ||
| eprintln!("error: no user profile directory for the gateway's run directory"); | ||
| return ExitCode::FAILURE; | ||
| }; | ||
|
|
||
| // The child's stdin stays open, unused, for as long as this process | ||
| // lives. When the pipe closes, the child's watchdog stops it. | ||
| let _child_stdin = child.stdin.take(); | ||
| let mut browser = options.serve.browser; | ||
|
|
||
| // The port the last child published. Every relaunch asks for it, so a | ||
| // restart keeps the Settings URL; the first child binds as configured. | ||
| let mut preferred_port = None; | ||
|
|
||
| // Launches made for the pending restart: 0 on the first boot and while | ||
| // a child serves. | ||
| let mut restart_launch_count = 0; | ||
|
|
||
| loop { | ||
| let child_options = { | ||
| let mut child_options = options.clone(); | ||
| child_options.serve = child_options | ||
| .serve | ||
| .with_browser(browser) | ||
| .with_preferred_port(preferred_port); | ||
| child_options | ||
| }; | ||
|
|
||
| let mut child = match SupervisedChild::spawn(&child_options, &run_dir) { | ||
| Ok(child) => child, | ||
| Err(error) => { | ||
| eprintln!("error: could not start the gateway: {error}"); | ||
| return ExitCode::FAILURE; | ||
|
Comment on lines
+62
to
+64
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Retry failed restart spawns within the restart budget. If a restart spawn fails, this branch ends the supervisor immediately. It bypasses the backoff and five-launch limit, even when the spawn error is temporary. The retry policy currently covers only children that spawn successfully. Keep immediate failure for the first boot. Route restart spawn errors through the capped retry path.
|
||
| } | ||
| }; | ||
|
|
||
| // Only the first launch opens the browser (or not), subsequent restarts never should. | ||
| browser = false; | ||
|
|
||
| let publication_result = match child.await_publication() { | ||
| Ok(Launch::Published(port)) => { | ||
| preferred_port = Some(port); | ||
|
|
||
| match child.wait_for_exit() { | ||
| Ok(status) if status.code() == Some(i32::from(RESTART_REQUESTED)) => { | ||
| restart_launch_count = 1; | ||
| continue; | ||
| } | ||
|
|
||
| Ok(status) => return exit_code(status), | ||
|
|
||
| Err(error) => { | ||
| eprintln!("error: could not wait for the gateway: {error}"); | ||
| return ExitCode::FAILURE; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // A child that exits 0 before publishing handed off to a Gateway | ||
| // it found serving; the supervisor ends with it. | ||
| Ok(Launch::Exited(status)) if status.success() => return ExitCode::SUCCESS, | ||
|
|
||
| Ok(Launch::Exited(status)) => exit_code(status), | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
If the child publishes, accepts a restart request, and exits before Prompt To Fix With AIThis is a comment left during a code review.
Path: crates/gateway/app/src/supervisor.rs
Line: 94
Comment:
**Fast restart stops the gateway**
If the child publishes, accepts a restart request, and exits before `await_publication` observes it, this branch passes through exit code 75 instead of restarting. This can happen during the 25 ms polling gap or while the supervisor is not scheduled. On first boot, the supervisor exits and leaves the gateway stopped. Handle `RESTART_REQUESTED` in this branch too.
---
For each issue above, determine whether it is valid and should be fixed. If so, fix it directly. |
||
|
|
||
| Ok(Launch::TimedOut) => { | ||
| eprintln!( | ||
| "error: the gateway published no connection within {PUBLICATION_TIMEOUT:?}; stopping it" | ||
| ); | ||
| child.kill(); | ||
| ExitCode::FAILURE | ||
| } | ||
|
|
||
| Err(error) => { | ||
| eprintln!("error: could not watch the gateway start: {error}"); | ||
| child.kill(); | ||
| ExitCode::FAILURE | ||
| } | ||
| }; | ||
|
|
||
| // A failed first boot is never retried. A failed restart launch is, | ||
| // until the restart has used all its launches. | ||
| if restart_launch_count == 0 || restart_launch_count >= MAX_RESTART_LAUNCHES { | ||
| return publication_result; | ||
| } | ||
|
|
||
| std::thread::sleep(backoff(restart_launch_count)); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How high can
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It can go up to |
||
| restart_launch_count += 1; | ||
| } | ||
| } | ||
|
|
||
| /// The wait after a restart's `failed`-th launch fails: 250 ms, then | ||
| /// doubling. | ||
| fn backoff(failed: u32) -> Duration { | ||
| BASE_BACKOFF_TIMEOUT * 2u32.pow(failed - 1) | ||
| } | ||
|
|
||
| /// What a launched child did first. | ||
| enum Launch { | ||
| /// It published its discovery file on this port and is serving. | ||
| Published(u16), | ||
|
|
||
| /// It exited before publishing. | ||
| Exited(ExitStatus), | ||
|
|
||
| /// It published nothing within the timeout and is still running. | ||
| TimedOut, | ||
| } | ||
|
|
||
| /// A running child Gateway, with its held-open stdin and the threads | ||
| /// forwarding its output. | ||
| struct SupervisedChild<'a> { | ||
| process: Child, | ||
|
|
||
| #[expect( | ||
| dead_code, | ||
| reason = "Needs to be held for the child's lifetime or the child's watchdog will kill it. We never write to it, so its unused." | ||
| )] | ||
| stdin: Option<ChildStdin>, | ||
|
|
||
| forwarders: Vec<JoinHandle<()>>, | ||
|
|
||
| run_dir: &'a PathBuf, | ||
| } | ||
|
|
||
| impl<'a> SupervisedChild<'a> { | ||
| fn spawn(options: &AppOptions, run_dir: &'a PathBuf) -> std::io::Result<Self> { | ||
| let mut process = build_child_command(options).spawn()?; | ||
|
|
||
| let stdin = process.stdin.take(); | ||
|
|
||
| let forwarders = [ | ||
| process | ||
| .stdout | ||
| .take() | ||
| .map(|pipe| OutputStream::Stdout.forward(pipe)), | ||
| process | ||
| .stderr | ||
| .take() | ||
| .map(|pipe| OutputStream::Stderr.forward(pipe)), | ||
| ] | ||
| .into_iter() | ||
| .flatten() | ||
| .collect(); | ||
|
|
||
| Ok(Self { | ||
| process, | ||
| stdin, | ||
| forwarders, | ||
| run_dir, | ||
| }) | ||
| } | ||
|
|
||
| let child_stdout = child.stdout.take(); | ||
| let child_stderr = child.stderr.take(); | ||
| /// Waits until the child publishes its discovery file, exits, or runs | ||
| /// out of time. | ||
| fn await_publication(&mut self) -> std::io::Result<Launch> { | ||
| let deadline = Instant::now() + PUBLICATION_TIMEOUT; | ||
|
|
||
| // Forward the child's stdout and std_error to this thread's output streams | ||
| let output_handles = [ | ||
| child_stdout.map(|pipe| OutputStream::Stdout.forward(pipe)), | ||
| child_stderr.map(|pipe| OutputStream::Stderr.forward(pipe)), | ||
| ]; | ||
| loop { | ||
| // Publication is checked first: a child that served and then | ||
| // exited to restart has left its record, so it reads as | ||
| // published even when it exited between two polls. | ||
| if let Ok(Some(file)) = GatewayDiscoveryFile::read(self.run_dir) | ||
| && file.pid == self.process.id() | ||
| { | ||
| return Ok(Launch::Published(file.port)); | ||
| } | ||
|
|
||
| let status = child.wait(); | ||
| // Checks if the process has already exited | ||
| if self.process.try_wait()?.is_some() { | ||
| return self.wait_for_exit().map(Launch::Exited); | ||
| } | ||
|
|
||
| // The pipes close when the child exits; joining flushes its last lines. | ||
| for handle in output_handles.into_iter().flatten() { | ||
| let _ = handle.join(); | ||
| if Instant::now() >= deadline { | ||
| return Ok(Launch::TimedOut); | ||
| } | ||
|
|
||
| std::thread::sleep(PUBLICATION_POLL); | ||
| } | ||
| } | ||
|
|
||
| match status { | ||
| Ok(status) => exit_code(status), | ||
| Err(error) => { | ||
| eprintln!("error: could not wait for the gateway: {error}"); | ||
| ExitCode::FAILURE | ||
| /// Waits for the child to exit and for its last output to be forwarded. | ||
| fn wait_for_exit(&mut self) -> std::io::Result<ExitStatus> { | ||
| let status = self.process.wait(); | ||
|
|
||
| // The pipes close when the child exits; joining flushes its last lines. | ||
| for forwarder in self.forwarders.drain(..) { | ||
| let _ = forwarder.join(); | ||
| } | ||
|
|
||
| status | ||
| } | ||
|
|
||
| /// Stops the child and waits for it. | ||
| fn kill(&mut self) { | ||
| let _ = self.process.kill(); | ||
| let _ = self.wait_for_exit(); | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -82,6 +255,10 @@ fn build_child_command(options: &AppOptions) -> Command { | |
| command.arg("--browser"); | ||
| } | ||
|
|
||
| if let Some(port) = options.serve.preferred_port { | ||
| command.arg("--preferred-port").arg(port.to_string()); | ||
| } | ||
|
|
||
| command | ||
| .stdin(Stdio::piped()) | ||
| .stdout(Stdio::piped()) | ||
|
|
@@ -124,3 +301,7 @@ fn exit_code(status: ExitStatus) -> ExitCode { | |
| .and_then(|code| u8::try_from(code).ok()) | ||
| .map_or(ExitCode::FAILURE, ExitCode::from) | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| #[path = "supervisor-tests.rs"] | ||
| mod tests; | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If a relaunch hits a temporary process or file-handle limit,
SupervisedChild::spawnfails and the supervisor exits immediately, even when restart attempts remain. The gateway stays stopped without using the remaining retries. Route spawn failures through the same capped backoff used for other failed restart launches, while keeping first-boot failures immediate.Prompt To Fix With AI