norrietaylor · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026
@@ -356,6 +356,16 @@ impl<FS: FileSystem> Cache<FS> {
         f(&*guard)
     }
 
+    /// Releases the read tracker, closing its locked `alog/<n>.v1` file.
+    ///
+    /// Read trackers aren't essential to operation (see construction); a
+    /// daemon about to unmount the filesystem holding the cache calls this so
+    /// the tracker's held-open fd doesn't pin the mount busy. Subsequent
+    /// reads simply stop being recorded.
+    pub fn release_read_tracker(&self) {
+        self.inner().read_tracker = None;
+    }
+
     /// Reads a directory cached as the given spec hash.
     pub fn read_dir(&self, hash: &SpecHash) -> Result<DirCacheEntry<FS>, CacheErr> {
         let i = self.inner();

@@ -263,6 +263,13 @@ impl DaemonContext {
         self.cache.clone()
     }
 
+    /// Releases the local cache's read tracker (its held-open append-log fd).
+    /// Called on daemon shutdown before unmounting the filesystem that holds
+    /// the cache; harmless otherwise (read tracking is best-effort).
+    pub fn release_cache_read_tracker(&self) {
+        self.cache.release_read_tracker();
+    }
+
     /// Initializes a bunch of internals and returns them. Use
     /// [`DaemonContext::init`] instead unless you are `minimal init`,
     /// which needs to use these before a full context can be built.

@@ -479,23 +479,50 @@ async fn async_main() -> Result<(), MainError> {
     // relocate cache + state onto it so builds hardlinking from the cache stay on
     // one filesystem (the EXDEV fix). Relocation is gated on the mount succeeding:
     // pointing state at an unmounted /var/lib/minimal would land it on the
-    // read-only rootfs. When no volume is requested (the transitional case), state
-    // stays on the tmpfs default. Unit 2 (R2.4/R2.5) makes a failed mount loud and
-    // removes the silent-fallback path.
+    // read-only rootfs.
+    //
+    // R2.4/R2.5: a mount failure is loud and terminal — emit MOUNT_FAILED
+    // instead of READY and park. No code path substitutes the /run/minimal
+    // tmpfs: session state is user data with no host copy, so a silent fallback
+    // would serve a ghost READY over a VM that quietly loses everything on stop.
+    let mut state_volume_mounted = false;
     if let Some(dev) = cli.listen_args().unwrap().mk_mount_state_volume.clone() {
-        match guest::mount_state_volume(&dev, "/var/lib/minimal") {
-            Ok(true) => {
-                cli.global_args.minimal_state_dir =
-                    Some(DaemonAbsPath::try_new("/var/lib/minimal").unwrap().into());
+        match guest::mount_state_volume(&dev, guest::STATE_VOLUME_MOUNTPOINT) {
+            Ok(()) => {
+                cli.global_args.minimal_state_dir = Some(
+                    DaemonAbsPath::try_new(guest::STATE_VOLUME_MOUNTPOINT)
+                        .unwrap()
+                        .into(),
+                );
                 cli.global_args.minimal_cache_dir = Some(
-                    DaemonAbsPath::try_new("/var/lib/minimal/cache")
+                    DaemonAbsPath::try_new(format!("{}/cache", guest::STATE_VOLUME_MOUNTPOINT))
                         .unwrap()
                         .into(),
                 );
+                state_volume_mounted = true;
                 tracing::info!(device = %dev, "cache + state relocated onto the data volume (/var/lib/minimal)");
             }
-            Ok(false) => {}
-            Err(e) => tracing::error!(error = %e, device = %dev, "mounting data volume"),
+            // The MOUNT_FAILED beacon + park contract only makes sense with a
+            // minvmd host watching the marker socket (the vsock transport);
+            // a native daemon handed --mk-mount-state-volume must fail like
+            // any other startup error instead of hanging forever.
+            Err(e) if cli.listen_args().unwrap().vsock => {
+                tracing::error!(error = %e, device = %dev, "data volume mount failed; refusing READY (R2.4)");
+                if let Err(emit) = guest::emit_mount_failed_marker(&e.to_string()).await {
+                    // The host will still fail this boot via its READY
+                    // timeout; it just loses the mount-failure diagnosis.
+                    tracing::error!(error = %emit, "emitting MOUNT_FAILED marker failed; host will see a READY timeout");
+                }
+                // Park like the no-rootfs degraded path above: exiting pid-1
+                // tears the VMM down racing the host's marker read; the host
+                // kills the child once it has consumed MOUNT_FAILED.
+                loop {
+                    tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
+                }
+            }
+            Err(e) => {
+                return Err(MainError::IO(e, "mounting the state volume"));
+            }
         }
     }
 
@@ -516,12 +543,26 @@ async fn async_main() -> Result<(), MainError> {
     // Single-instance guard, held for the daemon's lifetime (the kernel
     // releases it on death): a second minimald must not steal this
     // instance's socket.
+    //
+    // As the microVM's pid-1 the lock lives on the /run tmpfs, not the
+    // provider dir: the provider dir sits on the persistent data volume, and
+    // a lifetime-held write fd there pins the volume busy through the
+    // shutdown quiesce (R2.1), leaving a dirty ext4 journal on every clean
+    // stop. Nothing outside this boot reads the guest's lock (the host probes
+    // its own provider dir), so boot-ephemeral tmpfs is the honest home for
+    // it. Keyed on being the VM init — pid-1 owns its /run — NOT on the
+    // `--vsock` flag: a native (possibly non-root) `--vsock` daemon may not
+    // be able to write /run at all and keeps the provider-dir lock.
+    let instance_lock_path = if is_minimal_microvm() {
+        DaemonAbsPath::try_new(format!("/run/minimald-local-{}.lock", cli.instance_num()))
+            .expect("static /run lock path is absolute")
+    } else {
+        cli.client_instance_dir()
+            .sub_path_unchecked(paths::MINIMALD_LOCK_FILE)
+    };
     let mut instance_lock = fd_lock::RwLock::new(
-        open_lock_file(
-            cli.client_instance_dir()
-                .sub_path_unchecked(paths::MINIMALD_LOCK_FILE),
-        )
-        .map_err(|e| MainError::IO(e, "opening instance lock"))?,
+        open_lock_file(instance_lock_path)
+            .map_err(|e| MainError::IO(e, "opening instance lock"))?,
     );
     let instance_guard = match instance_lock.try_write() {
         Ok(guard) => guard,
@@ -571,6 +612,7 @@ async fn async_main() -> Result<(), MainError> {
         // `OwnIp` PTask must attach to the host gvproxy over the vsock shuttle,
         // not spawn gvproxy in-guest. The UDS path is DM2.
         in_microvm: cli.listen_args().unwrap().vsock,
+        state_volume_mounted,
     };
     // Ensure the SSH host key is accessible in a instance-specific known_hosts file.
     // R1.2: load once and reuse in the vsock beacon so there is no redundant disk read.

@@ -245,6 +245,12 @@ async fn serve_shutdown(s: ServerStateHandle, c: RuChannel<Msg>) {
             let mngr = s.sessions_manager().await;
             Ok(match mngr.shutdown(req.force).await {
                 Ok(()) => {
+                    // R2.1/R2.2: with the sessions drained, quiesce the state
+                    // volume (syncfs + detach) before acknowledging, so a
+                    // caller-driven VMM teardown right after the ack leaves a
+                    // clean ext4 journal. Best-effort with a bounded wait; the
+                    // journal replay backstop covers every failure arm.
+                    quiesce_state_volume_if_mounted(&s).await;
                     // Manager is down; tell the accept loop to stop and drain
                     // so the process can exit. Firing before the response is
                     // written is safe: the drain waits out the grace period,
@@ -261,6 +267,39 @@ async fn serve_shutdown(s: ServerStateHandle, c: RuChannel<Msg>) {
     }
 }
 
+/// Quiesce the guest state volume during shutdown (R2.2). No-op unless the
+/// boot path actually mounted the data volume at the state dir — a native
+/// daemon's host directory, or a microVM running without a volume, must
+/// never be synced-and-unmounted out from under the host. `syncfs` is
+/// blocking, so it runs on the blocking pool with a 10 s ceiling; the
+/// handler proceeds regardless of the outcome. Note the ceiling's residual
+/// risk: a timed-out `syncfs` keeps running detached while the handler acks,
+/// so a very large dirty set can still be mid-flush when the caller tears
+/// the VM down — bounded, as ever, by the ext4 journal replay backstop.
+#[cfg(target_os = "linux")]
+async fn quiesce_state_volume_if_mounted(s: &ServerStateHandle) {
+    const QUIESCE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
+
+    if !s.state_volume_mounted().await {
+        return;
+    }
+    let mountpoint = s.minimal_state_dir().await;
+    let quiesce = tokio::task::spawn_blocking(move || {
+        crate::guest::quiesce_state_volume(mountpoint.as_utf8_path().as_str())
+    });
+    match tokio::time::timeout(QUIESCE_TIMEOUT, quiesce).await {
+        Ok(Ok(Ok(()))) => tracing::info!("state volume quiesced for shutdown"),
+        Ok(Ok(Err(e))) => {
+            tracing::warn!(error = %e, "state volume quiesce failed; ext4 journal replay will recover")
+        }
+        Ok(Err(join)) => tracing::warn!(error = %join, "state volume quiesce task panicked"),
+        Err(_) => tracing::warn!("state volume quiesce timed out after 10s; proceeding"),
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+async fn quiesce_state_volume_if_mounted(_s: &ServerStateHandle) {}
+
 /// `AbortSession`: drop a `Pending` session's stash entry and delete
 /// its on-disk record. See the manager arm for the actor-side rules
 /// (refuses `Active` records and unknown ids).

@@ -50,6 +50,13 @@ pub struct Config {
     /// `false` (DM2, native Linux) keeps the local-spawn + tap relay path.
     #[serde(default)]
     pub in_microvm: bool,
+    /// Whether the guest boot path actually mounted the writable data volume
+    /// at `minimal_state_dir`. Gates the shutdown quiesce (R2.1/R2.2): only a
+    /// filesystem this daemon mounted may be synced and unmounted — the vsock
+    /// transport alone doesn't imply one (a native `--vsock` daemon, or a
+    /// microVM booted without a data volume, must never unmount its state dir).
+    #[serde(default)]
+    pub state_volume_mounted: bool,
 }
 
 impl Config {
@@ -229,6 +236,20 @@ impl ServerStateHandle {
         self.0.lock().await.shutdown.cancel();
     }
 
+    /// Whether the boot path mounted the writable data volume at the state
+    /// dir. The `Shutdown` RPC handler quiesces (syncs + unmounts) the state
+    /// dir only when this daemon mounted it — never a host directory or a
+    /// tmpfs it merely uses.
+    pub(crate) async fn state_volume_mounted(&self) -> bool {
+        self.0.lock().await.config.state_volume_mounted
+    }
+
+    /// The configured state dir (the quiesce target when in a microVM).
+    #[cfg(target_os = "linux")]
+    pub(crate) async fn minimal_state_dir(&self) -> DaemonAbsPath {
+        self.0.lock().await.config.minimal_state_dir.clone()
+    }
+
     /// Returns the daemon's TLS certificate authority (only with
     /// `networking-proxy` feature). Used by the `IssueClientCert` RPC handler.
     #[cfg(feature = "networking-proxy")]
@@ -600,6 +621,7 @@ mod tests {
             minimal_cache_dir: DaemonAbsPath::try_new(path).unwrap(),
             gvproxy_bin: None,
             in_microvm: false,
+            state_volume_mounted: false,
         }
     }
 

@@ -894,6 +894,11 @@ impl<L: Loader> Manager<L> {
                     // against a not-yet-existing "restart in place"
                     // flow.
                     self.compositions.clear();
+                    // Release the cache's held-open alog fd: it lives on the
+                    // state volume, and a surviving write-open fd there makes
+                    // the post-drain quiesce (R2.1 syncfs + unmount) fail
+                    // EBUSY, leaving the ext4 journal dirty on clean stops.
+                    self.daemon_ctx.release_cache_read_tracker();
                     Ok(Ok(()))
                 })
                 .await

@@ -57,6 +57,7 @@ impl TestServer {
             minimal_cache_dir: cache_dir,
             gvproxy_bin: None,
             in_microvm: false,
+            state_volume_mounted: false,
         };
         let state = ServerStateHandle::new(config).await.unwrap();
 

@@ -98,6 +98,13 @@ fn run_boot(foreground: bool) -> Result<()> {
         .set_nonblocking(false)
         .context("setting listener to blocking")?;
 
+    // R2.5: record whether the data volume image pre-exists this boot, before
+    // the VMM child provisions it — a later boot failure is fatal for a
+    // pre-existing image (may hold session data) and recoverable for a blank
+    // one freshly created by this boot.
+    let volume_path = crate::volume::resolve_data_volume_path();
+    let volume_preexisted = crate::cmd::volume_preexists(&volume_path);
+
     // Spawn `minvmd __krun-vmm` — the VMM child that calls krun_start_enter.
     let exe = std::env::current_exe().context("resolving current executable path")?;
     let mut cmd = std::process::Command::new(&exe);
@@ -124,10 +131,10 @@ fn run_boot(foreground: bool) -> Result<()> {
 
     // Run the accept loop in a separate thread so we can apply a wall-clock
     // timeout without platform-specific socket options.
-    let (tx, rx) = std::sync::mpsc::channel::<Result<(), String>>();
+    let (tx, rx) = std::sync::mpsc::channel::<Result<crate::cmd::BootBeacon, String>>();
     let sock_path_clone = marker_sock_path.clone();
     std::thread::spawn(move || {
-        let result = (|| -> Result<(), String> {
+        let result = (|| -> Result<crate::cmd::BootBeacon, String> {
             let (stream, _) = listener
                 .accept()
                 .map_err(|e| format!("accept on READY-marker socket: {e}"))?;
@@ -139,7 +146,7 @@ fn run_boot(foreground: bool) -> Result<()> {
     });
 
     match rx.recv_timeout(ready_timeout) {
-        Ok(Ok(())) => {
+        Ok(Ok(crate::cmd::BootBeacon::Ready)) => {
             println!("vm-up");
             // R3.2: by the time READY arrives, libkrun has created and is
             // listening on the minimald bridge socket. libkrun creates it with
@@ -165,16 +172,29 @@ fn run_boot(foreground: bool) -> Result<()> {
                 }
             }
         }
+        Ok(Ok(crate::cmd::BootBeacon::MountFailed { reason })) => {
+            let _ = child.kill();
+            let _ = child.wait();
+            let _ = std::fs::remove_file(&marker_sock_path);
+            crate::cmd::discard_fresh_volume_image(&volume_path, volume_preexisted);
+            return Err(crate::cmd::mount_failed_error(
+                &reason,
+                &volume_path,
+                volume_preexisted,
+            ));
+        }
         Ok(Err(e)) => {
             let _ = child.kill();
             let _ = child.wait();
             let _ = std::fs::remove_file(&marker_sock_path);
+            crate::cmd::discard_fresh_volume_image(&volume_path, volume_preexisted);
             bail!("boot failed: {e}");
         }
         Err(_timeout) => {
             let _ = child.kill();
             let _ = child.wait();
             let _ = std::fs::remove_file(&marker_sock_path);
+            crate::cmd::discard_fresh_volume_image(&volume_path, volume_preexisted);
             bail!(
                 "boot timed out waiting for READY marker after {} s (raise {} to wait longer)",
                 ready_timeout.as_secs(),