norrietaylor · Jul 9, 2026 · Jul 8, 2026 · Jul 8, 2026 · Jul 8, 2026 · Jul 8, 2026
@@ -148,6 +148,17 @@ jobs:
         # `minimal` CLI build it drives is incremental.
         run: ./scripts/fetch-libkrun.sh "$KRUN_PREFIX" x86_64
 
+      - name: Assert libkrun exports krun_add_disk3 (>= 1.19.0)
+        # krun_add_disk3 (this PR's /dev/vdb attach) needs libkrun >= 1.19.0. Fail
+        # loudly here rather than with a cryptic runtime error at VM boot if the
+        # materialized upstream libkrun is too old.
+        run: |
+          so="$(ls "$KRUN_PREFIX"/libkrun.so* 2>/dev/null | head -1)"
+          test -n "$so" || { echo "::error::no libkrun.so under $KRUN_PREFIX"; exit 1; }
+          nm -D "$so" 2>/dev/null | grep -q krun_add_disk3 \
+            || { echo "::error::materialized libkrun ($so) does not export krun_add_disk3; need >= 1.19.0"; exit 1; }
+          echo "libkrun OK: $so exports krun_add_disk3"
+
       - name: Materialize gvproxy switch binary (DM1 relay)
         # The DM1 gvproxy relay e2e (vsock_relay_e2e) spawns this per-host switch
         # binary; pin-verified, no Go toolchain. Publish its path for the gated

@@ -9,6 +9,7 @@ on:
         branches: ["main"]
         paths:
             - "crates/minvmd/**"
+            - "crates/minimald/**" # guest daemon: runs as pid-1 in the e2e/boot VM
             - "crates/minimal/**"
             - "crates/sessions/**" # platform-sensitive: canonicalization, symlinks
             - "Cargo.toml"
@@ -19,6 +20,7 @@ on:
         branches: ["main"]
         paths:
             - "crates/minvmd/**"
+            - "crates/minimald/**" # guest daemon: runs as pid-1 in the e2e/boot VM
             - "crates/minimal/**"
             - "crates/sessions/**" # platform-sensitive: canonicalization, symlinks
             - "Cargo.toml"
@@ -141,6 +143,20 @@ jobs:
               # Idempotent (a no-op when already present); same slp/krun tap the runner
               # setup used, so the supply-chain surface is unchanged.
               run: brew install slp/krun/libkrun
+            - name: Verify libkrun is available (>= 1.19.0)
+              # krun_add_disk3 (the /dev/vdb attach) needs libkrun >= 1.19.0. Assert
+              # the runner's libkrun exports it, so a stale runner fails here with an
+              # actionable message instead of a cryptic dyld symbol-not-found at boot.
+              run: |
+                  dylib=/opt/homebrew/lib/libkrun.dylib
+                  if [ ! -f "$dylib" ]; then
+                    echo "::error::libkrun.dylib not found under /opt/homebrew/lib. Provision the runner with: brew install slp/krun/libkrun" >&2
+                    exit 1
+                  fi
+                  if ! nm -gU "$dylib" 2>/dev/null | grep -q '_krun_add_disk3'; then
+                    echo "::error::runner libkrun lacks krun_add_disk3 (need >= 1.19.0); upgrade with: brew upgrade slp/krun/libkrun" >&2
+                    exit 1
+                  fi
             - name: Download virtio-linux kernel
               uses: actions/download-artifact@v8
               with:
@@ -186,6 +202,17 @@ jobs:
                   MINVMD_INITRAMFS="$RUNNER_TEMP/initramfs/initramfs.cpio" \
                   MINVMD_BOOT_LOG="$RUNNER_TEMP/boot.log" \
                     "$testbin" --include-ignored --nocapture --exact minimald_exec_over_bridge
+            - name: Reap the Session-E2E VM before auto-spawn
+              # The session harness (Guest::drop) kills its minvmd but not the
+              # detached `__krun-vmm` grandchild it spawned; that VM lingers into
+              # the Auto-spawn step and, as a second concurrent VM, wedges the
+              # host→guest bridge over libkrun's vsock (#588), so the cold
+              # `minimal ls` connection is never delivered to the guest accept
+              # loop. Reap it so Auto-spawn starts from the single-VM state it had
+              # when it was its own job (before the boot+autospawn jobs merged).
+              run: |
+                  pkill -f '__krun-vmm' 2>/dev/null || true
+                  sleep 1
             - name: Auto-spawn E2E (minimal ls cold/warm, R4.5)
               # The R4.5 CLI proof artifact, on real hardware. From a clean state
               # `minimal ls` must auto-spawn minvmd and return within 8 s;
@@ -297,6 +324,20 @@ jobs:
               # present); same tap the runner setup used, so the supply-chain surface
               # is unchanged.
               run: brew install slp/krun/libkrun
+            - name: Verify libkrun is available (>= 1.19.0)
+              # krun_add_disk3 (the /dev/vdb attach) needs libkrun >= 1.19.0. Assert
+              # the runner's libkrun exports it, so a stale runner fails here with an
+              # actionable message instead of a cryptic dyld symbol-not-found at boot.
+              run: |
+                  dylib=/opt/homebrew/lib/libkrun.dylib
+                  if [ ! -f "$dylib" ]; then
+                    echo "::error::libkrun.dylib not found under /opt/homebrew/lib. Provision the runner with: brew install slp/krun/libkrun" >&2
+                    exit 1
+                  fi
+                  if ! nm -gU "$dylib" 2>/dev/null | grep -q '_krun_add_disk3'; then
+                    echo "::error::runner libkrun lacks krun_add_disk3 (need >= 1.19.0); upgrade with: brew upgrade slp/krun/libkrun" >&2
+                    exit 1
+                  fi
             - name: Clippy (minvmd)
               run: cargo clippy -p minvmd --all-targets -- -D warnings
             - name: Test (minvmd)

@@ -1,7 +1,7 @@
 [upstream] # Source of software & tooling
 repo = "https://github.com/gominimal/pkg
10BC8
s"
 branch = "main"
-locked_commit = "23193970569b93e5e7a980aa598c3342c386d33f"
+locked_commit = "12f324d8f436bd014e54269b3ff412390d4040d5"
 
 [stack]
 use = "rust"

@@ -239,6 +239,155 @@ pub fn enter_rootfs(device: &str) -> std::io::Result<()> {
     Ok(())
 }
 
+/// ext4 superblock magic (`0xEF53`), stored little-endian at byte offset 1080
+/// (superblock starts at 1024; `s_magic` is at offset 56 within it).
+const EXT4_MAGIC_OFFSET: u64 = 1080;
+const EXT4_MAGIC_LE: [u8; 2] = [0x53, 0xEF];
+
+/// Whether `device` already carries an ext4 filesystem, probed by reading the
+/// superblock magic. This is the idempotency gate for [`mount_state_volume`]:
+/// `mkfs` runs only when the magic is absent, so a VM restart after a partial
+/// format re-formats cleanly and a formatted volume is reused untouched.
+fn has_ext4_superblock(device: &str) -> std::io::Result<bool> {
+    use std::io::{Read, Seek, SeekFrom};
+    let mut f = std::fs::File::open(device)?;
+    f.seek(SeekFrom::Start(EXT4_MAGIC_OFFSET))?;
+    let mut buf = [0u8; 2];
+    match f.read_exact(&mut buf) {
+        Ok(()) => Ok(buf == EXT4_MAGIC_LE),
+        // A device smaller than the superblock offset cannot hold ext4.
+        Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => Ok(false),
+        Err(e) => Err(e),
+    }
+}
+
+/// The device block size ext4 is formatted with (4 KiB).
+const EXT4_BLOCK_BYTES: u64 = 4096;
+
+/// Safety margin left below the device size when formatting (spec R1.5).
+///
+/// libkrun reserves a small trailer on the raw backing file and shaves it off
+/// between boots (observed: 64 KiB), so the block device the guest sees can
+/// shrink slightly after the first boot. Formatting to the *exact* device size
+/// then fails on the next boot with `EXT4-fs: bad geometry: block count exceeds
+/// device`. Sizing the filesystem 1 MiB below the device (16× the observed shave)
+/// keeps the ext4 geometry valid across reboots; 1 MiB on a multi-GiB volume is
+/// negligible.
+const MKFS_MARGIN_BYTES: u64 = 1024 * 1024;
+
+/// The size of `device` in bytes, read from `/sys/block/<name>/size` (which is
+/// in 512-byte sectors).
+fn device_size_bytes(device: &str) -> std::io::Result<u64> {
+    let name = std::path::Path::new(device)
+        .file_name()
+        .and_then(|n| n.to_str())
+        .ok_or_else(|| {
+            std::io::Error::new(std::io::ErrorKind::InvalidInput, "device has no basename")
+        })?;
+    let sectors: u64 = std::fs::read_to_string(format!("/sys/block/{name}/size"))?
+        .trim()
+        .parse()
+        .map_err(|e| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                format!("bad block size: {e}"),
+            )
+        })?;
+    Ok(sectors * 512)
+}
+
+/// Smallest device we will format. Below this, the usable size after
+/// [`MKFS_MARGIN_BYTES`] leaves too little for a journalled ext4 (and a device
+/// ≤ the margin yields a zero block count), so `run_mkfs_ext4` rejects it rather
+/// than handing `mkfs.ext4` a nonsensical size. The real data volume is
+/// GiB-scale, so this only guards a misconfigured `MINVMD_VOLUME_BYTES` or a
+/// malformed device.
+const MKFS_MIN_DEVICE_BYTES: u64 = 16 * 1024 * 1024;
+
+/// Format `device` as ext4 via `mkfs.ext4 -F` (non-interactive). Resolves from
+/// `PATH` (`/usr/sbin`, set post-chroot in [`enter_rootfs`]); requires
+/// `e2fsprogs` in the rootfs closure (spec R1.7).
+///
+/// ext4's defaults are a decade of hard-won tuning and are left alone, including
+/// the inode ratio: mke2fs already defaults to a ~65536 bytes/inode ratio at
+/// these volume sizes (measured — identical inode count with or without an
+/// explicit `-i`), so no `-i` override is passed. Two non-default choices remain:
+///
+/// 1. **Eager inode + journal init** (`-E lazy_itable_init=0,lazy_journal_init=0`):
+///    zero the inode table + journal *now*, at mkfs, rather than in the
+///    background `ext4lazyinit` kernel thread after mount, so no background init
+///    competes with the guest bringing up its vsock bridge + SSH server on 2
+///    vCPUs. It's cheap because the volume is sparse — the zero-writes land in
+///    unallocated holes, so the init is ~instant regardless of size (measured
+///    sub-ms at 32–256 GiB). It runs once — first boot only; later boots detect
+///    the superblock and skip mkfs.
+///
+/// 2. **Survive libkrun's backing-file trailer shave** across reboots: pass an
+///    explicit block count sized [`MKFS_MARGIN_BYTES`] below the device (found via
+///    a 3-boot persistence test — without the margin the volume fails to re-mount
+///    on the next boot). The count is in block-size units, so `-b`
+///    [`EXT4_BLOCK_BYTES`] pins that unit to keep the arithmetic exact.
+fn run_mkfs_ext4(device: &str) -> std::io::Result<()> {
+    let device_bytes = device_size_bytes(device)?;
+    if device_bytes < MKFS_MIN_DEVICE_BYTES {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!(
+                "data volume {device} is too small to format: {device_bytes} bytes \
+                 (minimum {MKFS_MIN_DEVICE_BYTES})"
+            ),
+        ));
+    }
+    let fs_blocks = device_bytes.saturating_sub(MKFS_MARGIN_BYTES) / EXT4_BLOCK_BYTES;
+    let status = std::process::Command::new("mkfs.ext4")
+        .arg("-F")
+        .arg("-q")
+        .args(["-b", &EXT4_BLOCK_BYTES.to_string()])
+        .args(["-E", "lazy_itable_init=0,lazy_journal_init=0"])
+        .arg(device)
+        .arg(fs_blocks.to_string())
+        .status()?;
+    if !status.success() {
+        return Err(std::io::Error::other(format!(
+            "mkfs.ext4 -F {device} ({fs_blocks} blocks) failed: {status}"
+        )));
+    }
+    Ok(())
+}
+
+/// Mount the per-VM writable data volume (spec R1.5).
+///
+/// Runs **after** [`enter_rootfs`] has chrooted into the rootfs, so `mkfs.ext4`
+/// resolves against the rootfs userland (the initramfs has no e2fsprogs) and the
+/// mount joins the live root mount namespace. Formats `device` on first boot
+/// (superblock-gated, idempotent) and mounts it read-write at `mountpoint` with
+/// `MS_NOATIME`.
+///
+/// Returns `Ok(true)` when the volume was mounted, `Ok(false)` when `device` is
+/// absent — the transitional legacy case where no data volume is attached and
+/// the caller keeps the tmpfs state dir. Unit 2 (R2.4/R2.5) makes an *attached*
+/// volume that fails to mount fatal; this Unit-1 form only reports the outcome.
+pub fn mount_state_volume(device: &str, mountpoint: &str) -> std::io::Result<bool> {
+    // `try_exists` distinguishes "absent" (Ok(false)) from a stat error (Err):
+    // a plain `exists()` reports a transiently-unstattable but attached device as
+    // absent, silently dropping to the ephemeral tmpfs and losing the volume.
+    if !std::path::Path::new(device).try_exists()? {
+        tracing::info!(device, "no data volume attached; keeping tmpfs state dir");
+        return Ok(false);
+    }
+    // Ensure the mountpoint exists before formatting: it must be present on the
+    // read-only rootfs (spec R1.7), so on a rootfs that predates R1.7 this fails
+    // with EROFS — better to surface that here than after a needless mkfs.
+    std::fs::create_dir_all(mountpoint)?;
+    if !has_ext4_superblock(device)? {
+        tracing::info!(device, "no ext4 superblock; formatting data volume");
+        run_mkfs_ext4(device)?;
+    }
+    raw_mount(device, mountpoint, "ext4", libc::MS_NOATIME)?;
+    tracing::info!(device, mountpoint, "mounted writable state volume");
+    Ok(true)
+}
+
 /// Mounts `/proc` and `/sys` if they are not already present.
 ///
 /// The kernel auto-mounts devtmpfs on `/dev`, but not these pseudo
@@ -538,6 +687,38 @@ fn mount_if_absent(target: &str, source: &str, fstype: &str) {
 mod tests {
     use super::*;
 
+    #[test]
+    fn ext4_superblock_probe_detects_magic() {
+        use std::io::{Seek, SeekFrom, Write};
+        let dir = std::env::temp_dir().join(format!("guest-sb-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+
+        // A file with the ext4 magic at offset 1080 probes positive.
+        let ext4 = dir.join("ext4.img");
+        {
+            let mut f = std::fs::File::create(&ext4).unwrap();
+            f.set_len(2048).unwrap();
+            f.seek(SeekFrom::Start(EXT4_MAGIC_OFFSET)).unwrap();
+            f.write_all(&EXT4_MAGIC_LE).unwrap();
+        }
+        assert!(has_ext4_superblock(ext4.to_str().unwrap()).unwrap());
+
+        // A zeroed image (freshly provisioned, unformatted) probes negative.
+        let blank = dir.join("blank.img");
+        std::fs::File::create(&blank)
+            .unwrap()
+            .set_len(2048)
+            .unwrap();
+        assert!(!has_ext4_superblock(blank.to_str().unwrap()).unwrap());
+
+        // A device too small to hold a superblock probes negative, not error.
+        let tiny = dir.join("tiny.img");
+        std::fs::File::create(&tiny).unwrap().set_len(64).unwrap();
+        assert!(!has_ext4_superblock(tiny.to_str().unwrap()).unwrap());
+
+        let _ = std::fs::remove_dir_all(&dir);
+    }
+
     #[tokio::test]
     async fn write_ready_beacon_formats_two_lines() {
         use russh::keys::{Algorithm, PrivateKey, key::safe_rng};

@@ -169,6 +169,14 @@ pub struct ListenArgs {
     #[clap(hide = true)]
     mount_rootfs: Option<String>,
 
+    /// Device to format-on-first-boot + mount as the writable state volume at
+    /// `/var/lib/minimal` when running as a microVM init (R1.5/R1.6). When set
+    /// and the mount succeeds, cache + state are relocated onto it. Only useful
+    /// as a VM init process; `None` leaves state on the tmpfs default.
+    #[arg(long)]
+    #[clap(hide = true)]
+    mk_mount_state_volume: Option<String>,
+
     /// Daemonize: spawn minimald in a new session (setsid) and return once the
     /// SSH socket accepts connections, or an 8s timeout elapses. Used by the
     /// `minimal` CLI to auto-start a native (DM2) daemon on Linux.
@@ -286,13 +294,14 @@ async fn async_main() -> Result<(), MainError> {
     // would indicate we are operating in a single-purpose micro-vm.
     //
     // If we are not the init process, we load our config from CLI args.
-    let cli = if is_minimal_microvm() {
+    let mut cli = if is_minimal_microvm() {
         Cli {
             command: Command::Run(ListenArgs {
                 instance_num: 0,
                 vsock: true,
                 mount_dev: true,
                 mount_rootfs: Some("/dev/vda".to_string()),
+                mk_mount_state_volume: Some("/dev/vdb".to_string()),
                 detach: false,
                 // In-VM (DM1/3/4) the PTask attaches to the host gvproxy over the
                 // vsock shuttle, so no in-guest gvproxy binary path is needed.
@@ -353,6 +362,31 @@ async fn async_main() -> Result<(), MainError> {
         }
     }
 
+    // R1.5/R1.6: when the microVM config requested a data volume
+    // (`mk_mount_state_volume`), format-on-first-boot + mount it and, on success,
+    // relocate cache + state onto it so builds hardlinking from the cache stay on
+    // one filesystem (the EXDEV fix). Relocation is gated on the mount succeeding:
+    // pointing state at an unmounted /var/lib/minimal would land it on the
+    // read-only rootfs. When no volume is requested (the transitional case), state
+    // stays on the tmpfs default. Unit 2 (R2.4/R2.5) makes a failed mount loud and
+    // removes the silent-fallback path.
+    if let Some(dev) = cli.listen_args().unwrap().mk_mount_state_volume.clone() {
+        match guest::mount_state_volume(&dev, "/var/lib/minimal") {
+            Ok(true) => {
+                cli.global_args.minimal_state_dir =
+                    Some(DaemonAbsPath::try_new("/var/lib/minimal").unwrap().into());
+                cli.global_args.minimal_cache_dir = Some(
+                    DaemonAbsPath::try_new("/var/lib/minimal/cache")
+                        .unwrap()
+                        .into(),
+                );
+                tracing::info!(device = %dev, "cache + state relocated onto the data volume (/var/lib/minimal)");
+            }
+            Ok(false) => {}
+            Err(e) => tracing::error!(error = %e, device = %dev, "mounting data volume"),
+        }
+    }
+
     if let Err(e) = std::fs::create_dir_all(cli.minimal_state_dir())
         && e.kind() != std::io::ErrorKind::AlreadyExists
     {
@@ -377,11 +411,14 @@ async fn async_main() -> Result<(), MainError> {
         },
         minimal_state_dir: cli.minimal_state_dir(),
         minimal_cache_dir: cli.minimal_cache_dir(),
-        gvproxy_bin: listen_args.gvproxy_bin.clone(),
+        // Re-borrow `listen_args` fresh here (as the vsock branch below does):
+        // the R1.6 relocation above takes `&mut cli`, which ends the original
+        // `listen_args` borrow, so it cannot be held across that mutation.
+        gvproxy_bin: cli.listen_args().unwrap().gvproxy_bin.clone(),
         // The vsock listen path is exactly the libkrun-VM (DM1/3/4) case: an
         // `OwnIp` PTask must attach to the host gvproxy over the vsock shuttle,
         // not spawn gvproxy in-guest. The UDS path is DM2.
-        in_microvm: listen_args.vsock,
+        in_microvm: cli.listen_args().unwrap().vsock,
     };
     // Ensure the SSH host key is accessible in a instance-specific known_hosts file.
     // R1.2: load once and reuse in the vsock beacon so there is no redundant disk read.
@@ -443,7 +480,7 @@ async fn async_main() -> Result<(), MainError> {
             }
         };
 
-        let port_num = DEFAULT_VSOCK_PORT_BASE + listen_args.instance_num;
+        let port_num = DEFAULT_VSOCK_PORT_BASE + cli.listen_args().unwrap().instance_num;
         let listener = VsockListener::bind(VsockAddr::new(VMADDR_CID_ANY, port_num))
             .map_err(|e| MainError::IO(e, "binding vsock port"))?;