norrietaylor · Jul 13, 2026 · Jul 13, 2026 · Jul 13, 2026 · Jul 13, 2026 · Jul 13, 2026
@@ -44,3 +44,6 @@ op.workspace = true
 [dev-dependencies]
 minimald = { path = "../minimald", features = ["test-support"] }
 tempfile.workspace = true
+# `test-util` (not in tokio's `full`) for `#[tokio::test(start_paused = true)]`:
+# lets the client's handshake-deadline test elapse in virtual time.
+tokio = { workspace = true, features = ["test-util"] }
@@ -54,6 +54,12 @@ const STOPPING_POLL_MS: u64 = 100;
 #[cfg(any(target_os = "macos", target_os = "linux"))]
 const STOPPING_WAIT_SECS: u64 = 6;
 
+/// Max time to wait, after the daemon acknowledges a `Shutdown`, for the VM to
+/// finish going down. Covers the guest's connection drain (5 s grace), its
+/// poweroff, and the supervisor's state write.
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+const STOPPED_WAIT_SECS: u64 = 20;
+
 /// What to do given the lifecycle read from the minvmd state.
 #[cfg(any(target_os = "macos", target_os = "linux"))]
 #[derive(Debug, PartialEq, Eq)]
@@ -164,6 +170,61 @@ pub fn ensure_minvmd_running(_minimal_dir: Option<&Path>) -> io::Result<()> {
     Ok(())
 }
 
+/// Wait for a shutting-down minvmd to reach a terminal lifecycle state.
+///
+/// The `Shutdown` RPC is acknowledged when the guest *starts* going down: the
+/// daemon still has to drain its connections and power the VM off, and the
+/// supervisor has to reap the VMM child, before the state file reads `Stopped`.
+/// Returning to the user before then hands the next command a `Running` state
+/// whose VM is already gone — it would connect to a dead bridge socket instead
+/// of spawning a fresh VM (#730).
+///
+/// Reads via `effective_state`, so a supervisor that died without writing
+/// `Stopped` still resolves rather than pinning the wait to its deadline.
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+pub fn wait_for_minvmd_stopped(minimal_dir: Option<&Path>) -> io::Result<()> {
+    let provider_dir = crate::client::resolve_provider_dir(minimal_dir)?;
+    let state_dir = minvmd::state::StateDir::new(provider_dir)?;
+
+    let deadline = Instant::now() + Duration::from_secs(STOPPED_WAIT_SECS);
+    loop {
+        if !state_dir.effective_state()?.lifecycle.is_active() {
+            tracing::debug!("minvmd has stopped");
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(io::Error::other(format!(
+                "the VM is still shutting down after {STOPPED_WAIT_SECS}s; \
+                 run `minvmd stop` to force it down"
+            )));
+        }
+        thread::sleep(Duration::from_millis(STOPPING_POLL_MS));
+    }
+}
+
+/// Wait for the daemon the CLI talks to to finish shutting down. Only the
+/// minvmd backend has a VM to tear down (and lifecycle state to observe); a
+/// native minimald just exits, so there is nothing to wait for.
+#[cfg(target_os = "macos")]
+pub fn wait_for_daemon_stopped(use_minvmd: bool, minimal_dir: Option<&Path>) -> io::Result<()> {
+    let _ = use_minvmd;
+    wait_for_minvmd_stopped(minimal_dir)
+}
+
+#[cfg(target_os = "linux")]
+pub fn wait_for_daemon_stopped(use_minvmd: bool, minimal_dir: Option<&Path>) -> io::Result<()> {
+    if use_minvmd {
+        return wait_for_minvmd_stopped(minimal_dir);
+    }
+    Ok(())
+}
+
+#[cfg(not(any(target_os = "macos", target_os = "linux")))]
+pub fn wait_for_daemon_stopped(use_minvmd: bool, minimal_dir: Option<&Path>) -> io::Result<()> {
+    let _ = (use_minvmd, minimal_dir);
+    Ok(())
+}
+
 /// Ensure the daemon the CLI will talk to is running, spawning it if needed.
 ///
 /// Backend selection:
@@ -234,8 +295,9 @@ fn ensure_minimald_running(socket_path: &Path, minimal_dir: Option<&Path>) -> io
 #[cfg(test)]
 #[cfg(any(target_os = "macos", target_os = "linux"))]
 mod tests {
-    use super::{Decision, classify};
+    use super::{Decision, classify, wait_for_minvmd_stopped};
     use minvmd::lifecycle::Lifecycle;
+    use minvmd::state::{State, StateDir};
 
     #[test]
     fn running_or_starting_needs_no_spawn() {
@@ -253,4 +315,39 @@ mod tests {
     fn stopping_waits_for_terminal_state() {
         assert_eq!(classify(Lifecycle::Stopping), Decision::WaitForStopping);
     }
+
+    /// The state dir a `--minimal-dir` override resolves to.
+    fn state_dir_for(minimal_dir: &std::path::Path) -> StateDir {
+        StateDir::new(crate::client::resolve_provider_dir(Some(minimal_dir)).unwrap()).unwrap()
+    }
+
+    #[test]
+    fn wait_returns_at_once_when_already_stopped() {
+        let tmp = tempfile::tempdir().unwrap();
+        state_dir_for(tmp.path())
+            .write_state(&State::stopped())
+            .unwrap();
+        wait_for_minvmd_stopped(Some(tmp.path())).expect("a stopped VM needs no waiting");
+    }
+
+    /// A supervisor that died without writing `Stopped` leaves `Running` behind
+    /// but drops the alive lock. That is a stopped VM, not one still going down:
+    /// the wait must resolve it, not sit out its deadline.
+    #[test]
+    fn wait_resolves_stale_running_state_from_a_dead_supervisor() {
+        let tmp = tempfile::tempdir().unwrap();
+        let state_dir = state_dir_for(tmp.path());
+        state_dir
+            .write_state(&State {
+                lifecycle: Lifecycle::Running,
+                vmm_pid: Some(999_999_999),
+                started_at: Some(0),
+            })
+            .unwrap();
+        wait_for_minvmd_stopped(Some(tmp.path())).expect("stale Running must resolve as stopped");
+        assert_eq!(
+            state_dir.read_state().unwrap().lifecycle,
+            Lifecycle::Stopped
+        );
+    }
 }
@@ -16,6 +16,15 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
 const CONNECT_RETRIES: u32 = 20;
 /// Delay between connection retries.
 const CONNECT_RETRY_DELAY: Duration = Duration::from_millis(100);
+/// Deadline for the SSH handshake + auth once the socket has accepted.
+///
+/// A connect is not proof that anyone is home: on the VM backend the socket is
+/// libkrun's bridge, which accepts even when the guest behind it is wedged, so
+/// only a completed handshake proves there is a live server. Without a deadline
+/// the CLI blocks there forever (#730). Generous — a healthy endpoint answers in
+/// milliseconds, so this only bounds the pathological case. Mirrors `minvmd`'s
+/// own client, which guards the same bridge.
+const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
 
 /// russh client handler that accepts any ephemeral host key.
 ///
@@ -45,7 +54,8 @@ impl Client {
     ///
     /// Retries the UDS connect for up to ~2 seconds to absorb the post-boot
     /// race on macOS where the libkrun bridge UDS appears slightly after the
-    /// `vm-up` line.
+    /// `vm-up` line, and bounds the handshake by [`HANDSHAKE_TIMEOUT`] so a
+    /// wedged daemon behind an accepting socket fails instead of hanging.
     pub async fn connect(sock_path: &Path) -> Result<Self, anyhow::Error> {
         let stream = {
10BC8
             let mut conn = None;
@@ -72,18 +82,30 @@ impl Client {
         };
 
         let config = Arc::new(russh::client::Config::default());
-        let mut handle = russh::client::connect_stream(config, stream, MinimalClientHandler)
-            .await
-            .context("ssh connect")?;
+        let handshake = async {
+            let mut handle = russh::client::connect_stream(config, stream, MinimalClientHandler)
+                .await
+                .context("ssh connect")?;
+
+            let auth = handle
+                .authenticate_none("minimal-cli")
+                .await
+                .context("authenticate")?;
+
+            if !auth.success() {
+                return Err(anyhow::anyhow!("authentication rejected by daemon"));
+            }
+            Ok(handle)
+        };
 
-        let auth = handle
-            .authenticate_none("minimal-cli")
+        let handle = tokio::time::timeout(HANDSHAKE_TIMEOUT, handshake)
             .await
-            .context("authenticate")?;
-
-        if !auth.success() {
-            return Err(anyhow::anyhow!("authentication rejected by daemon"));
-        }
+            .map_err(|_| {
+                anyhow::anyhow!(
+                    "connect to {}: SSH handshake timed out after {HANDSHAKE_TIMEOUT:?}",
+                    sock_path.display()
+                )
+            })??;
 
         Ok(Client { handle })
     }
@@ -160,7 +182,7 @@ pub fn resolve_socket_path(
 
 #[cfg(test)]
 mod tests {
-    use super::resolve_socket_path;
+    use super::{Client, resolve_socket_path};
     use std::path::Path;
 
     #[test]
@@ -171,4 +193,22 @@ mod tests {
             Path::new("/tmp/minimal-test/providers/local-0/ssh.sock")
         );
     }
+
+    /// A socket that accepts but never speaks SSH — a wedged guest behind
+    /// libkrun's always-accepting bridge — must fail at the handshake deadline
+    /// instead of blocking the CLI forever (#730).
+    #[tokio::test(start_paused = true)]
+    async fn connect_fails_at_the_handshake_deadline_on_a_mute_listener() {
+        let tmp = tempfile::tempdir().unwrap();
+        let sock = tmp.path().join("mute.sock");
+        let _listener = tokio::net::UnixListener::bind(&sock).unwrap();
+
+        // `start_paused` auto-advances time while nothing is runnable, so the
+        // deadline elapses in virtual time — the test does not wait it out.
+        let err = Client::connect(&sock).await.map(|_| ()).unwrap_err();
+        assert!(
+            err.to_string().contains("SSH handshake timed out"),
+            "expected a handshake-deadline error, got: {err:#}"
+        );
+    }
 }
@@ -1131,7 +1131,21 @@ pub async fn cmd_stop(global: &GlobalArgs, args: StopArgs) -> Result<(), anyhow:
 
     match resp {
         ShutdownResponse::ShuttingDown => {
+            // Drop our connection before waiting: the daemon holds the shutdown
+            // open for its drain grace period while a client is still attached,
+            // and we are that client.
+            drop(client);
             println!("Daemon is shutting down.");
+            // The wait polls the lifecycle file on a sleep loop, so it goes on
+            // the blocking pool rather than stalling an async worker for up to
+            // 20s (rust-coding-standards: no blocking in an async context).
+            let (use_minvmd, minimal_dir) = (global.minvmd, global.minimal_dir.clone());
+            tokio::task::spawn_blocking(move || {
+                autospawn::wait_for_daemon_stopped(use_minvmd, minimal_dir.as_deref())
+            })
+            .await
+            .context("The wait for the daemon to stop panicked")?
+            .context("Failed while waiting for the daemon to stop")?;
             Ok(())
         }
         ShutdownResponse::SessionsLive => {

@@ -548,6 +548,47 @@ pub fn quiesce_state_volume(mountpoint: &str) -> std::io::Result<()> {
     Ok(())
 }
 
+/// Take the microVM down, by resetting it. Does not return on success: the
+/// guest resets, and libkrun's VMM exits with it. On failure it returns the
+/// `reboot(2)` error.
+///
+/// ONLY call this as the microVM's pid-1 (guard with [`is_minimal_microvm`]):
+/// on an ordinary host it reboots the machine.
+///
+/// This is how the guest ends. As pid-1 minimald has nothing to return to:
+/// letting `main` exit kills init, which panics the kernel ("Attempted to kill
+/// init!") — and with no `panic=` on the cmdline the kernel then spins in the
+/// panic handler forever. The VMM stays alive on a dead guest, the host keeps
+/// reporting the VM as `Running`, and every later CLI command blocks on a
+/// bridge socket nothing is behind (#730). Taking the VM down instead lets the
+/// supervisor reap the VMM child and mark it `Stopped`, so the next command
+/// boots a fresh one.
+///
+/// Reset (`RB_AUTOBOOT`), not power-off: a guest-initiated reset is what makes
+/// a firecracker-family VMM exit, and it is the only mechanism that works on
+/// both arches. `RB_POWER_OFF` needs a `pm_power_off` handler, which the x86_64
+/// guest kernel does not have — there the kernel logs "Power off not available:
+/// System halted instead" and halts the vCPU with the VMM still alive, which is
+/// #730 all over again. (aarch64 does have one, via PSCI `SYSTEM_OFF`, which is
+/// what hid the gap until CI's x86_64 KVM lane.) Reset reaches
+/// `KVM_EXIT_SHUTDOWN` on both: PSCI `SYSTEM_RESET` on aarch64, i8042 reset /
+/// triple fault on x86_64. libkrun exits on it rather than restarting the
+/// guest, so this ends the VM despite the name.
+///
+/// `sync(2)` first: `reboot(2)` does not flush filesystems, and this also runs
+/// on the paths that never reached the shutdown quiesce (a failed
+/// `Server::run`).
+pub fn shut_down_vm() -> std::io::Error {
+    // SAFETY: neither call takes pointers. `reboot(2)` needs CAP_SYS_BOOT,
+    // which the microVM's pid-1 has, and does not return on success; on failure
+    // it returns -1 with errno set, which the caller surfaces.
+    unsafe {
+        libc::sync();
+        libc::reboot(libc::RB_AUTOBOOT);
+    }
+    std::io::Error::last_os_error()
+}
+
 /// Enumerate open fds (and cwds) under `mountpoint` across all processes, for
 /// the EBUSY diagnostics in [`quiesce_state_volume`] — an unmountable volume
 /// is invisible without knowing who holds it. Best-effort `/proc` scan.

@@ -230,7 +230,33 @@ fn main() -> Result<(), MainError> {
         .enable_all()
         .build()
         .unwrap();
-    runtime.block_on(async_main())
+    let result = runtime.block_on(async_main());
+
+    // As the microVM's pid-1 we must not return: exiting init panics the guest
+    // kernel and wedges the VM (#730). Take the VM down instead — a clean
+    // shutdown (the `Shutdown` RPC drained the server) and a failed one alike,
+    // since either way there is no init left to run. Diverges on success.
+    #[cfg(target_os = "linux")]
+    if is_minimal_microvm() {
+        match &result {
+            Ok(()) => tracing::info!("microVM init finished; shutting the VM down"),
+            Err(e) => tracing::error!(error = ?e, "microVM init failed; shutting the VM down"),
+        }
+        let error = minimald::guest::shut_down_vm();
+        // Unreachable in practice — `reboot(2)` only fails for a caller without
+        // CAP_SYS_BOOT, and the microVM's pid-1 has it. But falling through to
+        // `return result` would exit init and panic the guest kernel, which is
+        // the wedge #730 is about; never trade one wedge for another. Park
+        // instead, as the boot path's degraded arms do: the kernel stays alive
+        // and idle (no panic-handler spin), the console keeps working, and
+        // `minvmd stop`'s SIGTERM can still reap the VMM.
+        tracing::error!(%error, "shutting the VM down failed; parking pid-1 (exiting it would panic the guest kernel)");
+        loop {
+            std::thread::sleep(std::time::Duration::from_secs(3600));
+        }
+    }
+
+    result
 }
 
 /// Re-exec this binary in a new session (`setsid`) with `--detach` stripped, so
@@ -689,9 +715,54 @@ async fn async_main() -> Result<(), MainError> {
     }
 }
 
+/// Whether this process is the microVM's init: the kernel runs the initramfs
+/// `/init` (this binary) as pid-1.
+///
+/// Both halves are load-bearing, because this now also gates `reboot(2)` (see
+/// [`minimald::guest::shut_down_vm`]). `argv[0]` is caller-controlled — a host
+/// could run `exec -a init minimald`, and with `CAP_SYS_BOOT` that would reset
+/// the machine on exit — so it cannot be trusted alone. pid-1 cannot be spoofed
+/// from userspace, but a native daemon running as a container's init would
+/// satisfy it, so it is not sufficient alone either. Only the microVM's init
+/// satisfies both.
 fn is_minimal_microvm() -> bool {
-    std::env::args_os()
-        .next()
-        .map(|a0| std::path::Path::new(&a0).file_name() == Some(std::ffi::OsStr::new("init")))
-        .unwrap_or(false)
+    is_microvm_init(std::process::id(), std::env::args_os().next().as_deref())
+}
+
+/// Pure form of [`is_minimal_microvm`], so the spoofing cases are testable —
+/// neither a process's pid nor its `argv[0]` can be set from within a test.
+fn is_microvm_init(pid: u32, argv0: Option<&std::ffi::OsStr>) -> bool {
+    pid == 1
+        && argv0
+            .map(|a0| std::path::Path::new(a0).file_name() == Some(std::ffi::OsStr::new("init")))
+            .unwrap_or(false)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::is_microvm_init;
+    use std::ffi::OsStr;
+
+    #[test]
+    fn the_microvm_init_is_pid_1_named_init() {
+        assert!(is_microvm_init(1, Some(OsStr::new("/init"))));
+        assert!(is_microvm_init(1, Some(OsStr::new("init"))));
+    }
+
+    /// The guard gates `reboot(2)`: a host process that merely *claims* to be
+    /// init (`exec -a init minimald`) must not reach it.
+    #[test]
+    fn a_spoofed_argv0_on_the_host_is_not_the_microvm_init() {
+        assert!(!is_microvm_init(4242, Some(OsStr::new("/init"))));
+        assert!(!is_microvm_init(4242, Some(OsStr::new("init"))));
+    }
+
+    /// pid-1 alone is not enough either: a native daemon can be a container's
+    /// init, and it must keep exiting normally rather than resetting the box.
+    #[test]
+    fn pid_1_under_another_name_is_not_the_microvm_init() {
+        assert!(!is_microvm_init(1, Some(OsStr::new("/usr/bin/minimald"))));
+        assert!(!is_microvm_init(1, Some(OsStr::new("minimald"))));
+        assert!(!is_microvm_init(1, None));
+    }
 }