norrietaylor · Jun 4, 2026 · Jun 4, 2026 · Jun 4, 2026 · Jun 4, 2026 · Jun 4, 2026
@@ -12,13 +12,15 @@ on:
       - "Cargo.toml"
       - "Cargo.lock"
       - ".github/workflows/ci-macos.yml"
+      - "scripts/**"
   pull_request:
     branches: ["main"]
     paths:
       - "crates/minvmd/**"
       - "Cargo.toml"
       - "Cargo.lock"
       - ".github/workflows/ci-macos.yml"
+      - "scripts/**"
   workflow_dispatch:
 
 env:
@@ -34,6 +36,93 @@ permissions:
   contents: read
 
 jobs:
+  virtio-kernel:
+    # Pull the prebuilt virtio-linux kernel (vmlinuz / Image.gz) from the public
+    # minimal build cache — no kernel build, no GCS auth — and hand it to the
+    # macOS boot job. The packaging `minimal` CLI is Linux-only, so this runs on
+    # Linux. Cheap (a cache pull), so it is unconditional and decoupled from the
+    # self-hosted runner.
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Fetch virtio-linux kernel from the build cache
+        run: ./scripts/fetch-virtio-kernel.sh "$RUNNER_TEMP/vmlinuz" aarch64
+      - name: Upload kernel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: virtio-kernel-aarch64
+          path: ${{ runner.temp }}/vmlinuz
+          if-no-files-found: error
+
+  boot-e2e:
+    # Boot E2E (R2.4 READY round-trip) on real hardware, using the cache-pulled
+    # kernel and the real minvmd guest rootfs (Alpine + READY-writer stub).
+    # Separate from build-macos so a kernel-fetch hiccup never blocks the
+    # always-green clippy/test/smoke checks.
+    #
+    # GATING: green on the runner — kernel PE_GZ load, virtio-fs root,
+    # /init.krun exec of the stub, and the guest READY connect-out on vsock 7350
+    # (krun_add_vsock_port == listen=false; host + guest both 7350). See #326.
+    if: ${{ vars.RUN_MACOS_CI != 'false' }}
+    needs: virtio-kernel
+    runs-on: [self-hosted, macOS, ARM64]
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Verify libkrun is available
+        run: |
+          if [ ! -f /opt/homebrew/lib/libkrun.dylib ]; then
+            echo "::error::libkrun.dylib not found under /opt/homebrew/lib. Provision the runner with: brew install slp/krun/libkrun" >&2
+            exit 1
+          fi
+      - name: Download virtio-linux kernel
+        uses: actions/download-artifact@v4
+        with:
+          name: virtio-kernel-aarch64
+          path: ${{ runner.temp }}/kernel
+      - name: Stage minvmd guest rootfs
+        # Real R2.2 rootfs: fetch the pinned Alpine minirootfs and overlay socat
+        # + the READY-writer stub (/sbin/minvmd-stub-init) via build-rootfs.sh.
+        # A setup failure fails the job — it is not hidden behind the non-gating
+        # test below.
+        run: scripts/build-rootfs.sh
+      - name: Build boot E2E + minvmd (no run)
+        # Build the test harness + minvmd bin, then codesign, then run the test
+        # BINARY DIRECTLY (below). Invoking `cargo test` again after signing
+        # relinks (and unsigns) target/debug/minvmd, so krun_start_enter fails
+        # with EINVAL. A direct-binary run never touches the signature.
+        run: cargo test -p minvmd --test boot_e2e --no-run
+      - name: Codesign minvmd (hypervisor entitlement, R1.4)
+        run: codesign --entitlements crates/minvmd/minvmd.entitlements --force -s - target/debug/minvmd
+      - name: Boot E2E (READY round-trip)
+        # Gating: a real Alpine microVM must boot and the guest must write READY
+        # over vsock 7350 within the budget. Also validates the virtio-linux
+        # kernel config (virtio-MMIO / VIRTIO_FS / VSOCK / HVC).
+        run: |
+          testbin="$(ls -1t target/debug/deps/boot_e2e-* | grep -v '\.d$' | head -1)"
+          test -x "$testbin" || { echo "::error::boot_e2e test binary not found"; exit 1; }
+          MINVMD_E2E=1 \
+          MINVMD_KERNEL_PATH="$RUNNER_TEMP/kernel/vmlinuz" \
+          MINVMD_ROOTFS_PATH="$(scripts/build-rootfs.sh --print-path)" \
+          MINVMD_BOOT_LOG="$RUNNER_TEMP/boot.log" \
+            "$testbin" --include-ignored --nocapture
+      - name: Upload guest boot console log
+        # Capture hvc0 output for diagnosing a stuck/failed boot (e.g. a missing
+        # virtio driver in the kernel). Always runs so a red boot is debuggable.
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: minvmd-boot-log
+          path: ${{ runner.temp }}/boot.log
+          if-no-files-found: ignore
+
   build-macos:
     # Self-hosted Apple Silicon runner. minvmd links libkrun
     # (`#[link(name = "krun")]`) and only compiles on macOS — the Linux CI

@@ -17,4 +17,5 @@ tracing.workspace = true
 tracing-subscriber.workspace = true
 
 [dev-dependencies]
+serial_test.workspace = true
 tempfile.workspace = true
@@ -0,0 +1,149 @@
+//! `minvmd boot` subcommand (R2.3, R2.4).
+//!
+//! The parent process:
+//! 1. Resolves kernel and rootfs paths from env vars (fail-fast validation).
+//! 2. Creates a UNIX socket at a unique temp path and begins listening (the
+//!    READY-marker socket).
+//! 3. Fork-execs `minvmd __krun-vmm` with `MINVMD_MARKER_SOCK` pointing to
+//!    that socket so the VMM child can register it with libkrun.
+//! 4. Writes the child PID to `vmm.pid` in the state directory.
+//! 5. Waits up to 5 s for the guest to connect and write `READY\n` on the
+//!    marker socket (R2.4). On success prints `vm-up` to stdout.
+//! 6. With `--foreground`: stays alive until the VMM child exits, propagating
+//!    its exit code.
+//!
+//! On Linux this subcommand bails immediately with a "macOS only" error so the
+//! Linux-only CI stays green.
+
+use anyhow::{Result, bail};
+
+/// Run the `boot` subcommand.
+///
+/// `foreground`: if true, block until the VMM child process exits after the VM
+/// is confirmed up.
+pub fn run(foreground: bool) -> Result<()> {
+    #[cfg(target_os = "macos")]
+    return run_macos(foreground);
+
+    #[cfg(not(target_os = "macos"))]
+    {
+        let _ = foreground;
+        bail!("`minvmd boot` is macOS-only; this Linux build is a no-op stub");
+    }
+}
+
+#[cfg(target_os = "macos")]
+fn run_macos(foreground: bool) -> Result<()> {
+    use std::io::{BufRead, BufReader};
+    use std::os::unix::net::UnixListener;
+    use std::path::PathBuf;
+    use std::time::Duration;
+
+    use anyhow::Context as _;
+
+    use crate::cmd::MARKER_SOCK_ENV;
+    use crate::image::{resolve_kernel_path, resolve_rootfs_path};
+    use crate::state::StateDir;
+
+    // Fail-fast: resolve paths before spawning anything.
+    let _kernel = resolve_kernel_path().context("resolving kernel path")?;
+    let _rootfs = resolve_rootfs_path().context("resolving rootfs path")?;
+
+    // Create the marker socket under /tmp with a PID + random nonce to prevent
+    // predictable-path TOCTOU attacks (local-only risk, but cheap to harden).
+    let nonce: u32 = {
+        use std::io::Read as _;
+        let mut buf = [0u8; 4];
+        std::fs::File::open("/dev/urandom")
+            .and_then(|mut f| f.read_exact(&mut buf))
+            .context("reading /dev/urandom for marker socket nonce")?;
+        u32::from_le_bytes(buf)
+    };
+    let marker_sock_path = PathBuf::from(format!(
+        "/tmp/minvmd-marker-{}-{nonce:08x}.sock",
+        std::process::id()
+    ));
+
+    // Remove any stale socket from a previous run.
+    let _ = std::fs::remove_file(&marker_sock_path);
+
+    let listener =
+        UnixListener::bind(&marker_sock_path).context("binding READY-marker unix socket")?;
+    listener
+        .set_nonblocking(false)
+        .context("setting listener to blocking")?;
+
+    // Spawn `minvmd __krun-vmm` — the VMM child that calls krun_start_enter.
+    let exe = std::env::current_exe().context("resolving current executable path")?;
+    let mut child = std::process::Command::new(&exe)
+        .arg("__krun-vmm")
+        .env(MARKER_SOCK_ENV, &marker_sock_path)
+        // Inherit MINVMD_KERNEL_PATH and MINVMD_ROOTFS_PATH from environment.
+        .spawn()
+        .with_context(|| format!("spawning VMM child: {}", exe.display()))?;
+
+    let child_pid = child.id();
+    tracing::info!(pid = child_pid, "VMM child spawned");
+
+    // Write vmm.pid to the state directory (R2.3).
+    let state_dir = StateDir::new(StateDir::default_path()).context("opening state dir")?;
+    let vmm_pid_path = state_dir.vmm_pid_path();
+    std::fs::write(&vmm_pid_path, format!("{child_pid}\n")).context("writing vmm.pid")?;
+
+    // Wait for the READY marker from the guest (R2.4): up to 5 s.
+    const READY_TIMEOUT: Duration = Duration::from_secs(5);
+
+    // Run the accept loop in a separate thread so we can apply a wall-clock
+    // timeout without platform-specific socket options.
+    let (tx, rx) = std::sync::mpsc::channel::<Result<(), String>>();
+    let sock_path_clone = marker_sock_path.clone();
+    std::thread::spawn(move || {
+        let result = (|| -> Result<(), String> {
+            let (stream, _) = listener
+                .accept()
+                .map_err(|e| format!("accept on READY-marker socket: {e}"))?;
+            let mut reader = BufReader::new(stream);
+            let mut line = String::new();
+            reader
+                .read_line(&mut line)
+                .map_err(|e| format!("reading READY marker: {e}"))?;
+            let trimmed = line.trim();
+            if trimmed != "READY" {
+                return Err(format!("expected READY on marker socket, got {trimmed:?}"));
+            }
+            Ok(())
+        })();
+        let _ = tx.send(result);
+        let _ = std::fs::remove_file(&sock_path_clone);
+    });
+
+    match rx.recv_timeout(READY_TIMEOUT) {
+        Ok(Ok(())) => {
+            println!("vm-up");
+        }
+        Ok(Err(e)) => {
+            let _ = child.kill();
+            let _ = child.wait();
+            let _ = std::fs::remove_file(&vmm_pid_path);
+            let _ = std::fs::remove_file(&marker_sock_path);
+            bail!("boot failed: {e}");
+        }
+        Err(_timeout) => {
+            let _ = child.kill();
+            let _ = child.wait();
+            let _ = std::fs::remove_file(&vmm_pid_path);
+            let _ = std::fs::remove_file(&marker_sock_path);
+            bail!("boot timed out waiting for READY marker after 5 s");
+        }
+    }
+
+    if foreground {
+        let status = child.wait().context("waiting for VMM child")?;
+        if !status.success() {
+            let code = status.code().unwrap_or(-1);
+            bail!("VMM child exited with code {code}");
+        }
+    }
+
+    Ok(())
+}
@@ -0,0 +1,21 @@
+//! CLI subcommand implementations for `minvmd`.
+
+pub mod boot;
+pub mod vmm_child;
+
+/// vsock port used by the VMM child to signal the parent that the guest is up.
+///
+/// The guest's init writes `READY\n` to this vsock port; libkrun forwards the
+/// connection to the host UNIX socket registered by the parent before spawning
+/// the VMM child. The parent reads `READY` to confirm boot (R2.4).
+///
+/// Must match the guest rootfs's READY/agent port. The canonical value is
+/// `7350` — both the guest rootfs manifest (`etc/minvmd/manifest`:
+/// `vsock_port_ready=7350`) and the reference impl (min-ctl: cold boot to a TCP
+/// connection on `127.0.0.1:7350`) use it. The host previously used `9799`,
+/// which no guest emits on, so the READY marker never arrived.
+pub const VSOCK_MARKER_PORT: u32 = 7350;
+
+/// Environment variable that carries the host-side UNIX socket path for the
+/// READY marker from the parent to the VMM child.
+pub const MARKER_SOCK_ENV: &str = "MINVMD_MARKER_SOCK";
@@ -0,0 +1,97 @@
+//! `minvmd __krun-vmm` hidden subcommand (R2.3).
+//!
+//! This is the VMM child process that calls `krun_start_enter`. It is spawned
+//! by `minvmd boot` and is not intended to be run directly by users (hence the
+//! double-underscore prefix).
+//!
+//! On macOS:
+//! 1. Reads kernel and rootfs from `MINVMD_KERNEL_PATH` / `MINVMD_ROOTFS_PATH`.
+//! 2. Reads the READY-marker socket path from `MINVMD_MARKER_SOCK`.
+//! 3. Creates and configures a libkrun context (kernel, rootfs, resources).
+//! 4. Sets the guest workload via `krun_set_exec` (libkrun's `/init.krun` runs
+//!    as PID 1 and forks it); default `/sbin/minvmd-stub-init`, `MINVMD_EXEC`
+//!    overrides.
+//! 5. Registers the marker socket for `VSOCK_MARKER_PORT` (guest→host): the
+//!    guest workload connects to that vsock port and writes `READY\n`, which
+//!    libkrun bridges to the host UNIX socket where the parent listens (R2.4).
+//! 6. Calls `krun_start_enter`, which boots the VM. On success libkrun
+//!    `exit()`s with the guest workload's exit code and never returns here.
+//!
+//! On Linux this subcommand bails immediately with a "macOS only" error.
+
+use anyhow::{Result, bail};
+
+/// Run the `__krun-vmm` subcommand.
+pub fn run() -> Result<()> {
+    #[cfg(target_os = "macos")]
+    return run_macos();
+
+    #[cfg(not(target_os = "macos"))]
+    bail!("`minvmd __krun-vmm` is macOS-only; this Linux build is a no-op stub");
+}
+
+#[cfg(target_os = "macos")]
+fn run_macos() -> Result<()> {
+    use anyhow::Context as _;
+
+    use crate::cmd::{MARKER_SOCK_ENV, VSOCK_MARKER_PORT};
+    use crate::image::{resolve_kernel_path, resolve_rootfs_path};
+    use crate::krun::Context;
+    use crate::vm::VmConfig;
+
+    let kernel = resolve_kernel_path().context("resolving kernel path")?;
+    let rootfs = resolve_rootfs_path().context("resolving rootfs path")?;
+    let marker_sock = std::env::var(MARKER_SOCK_ENV).with_context(|| {
+        format!("reading {MARKER_SOCK_ENV}: VMM child must be spawned by `minvmd boot`")
+    })?;
+
+    let mut ctx = Context::create().context("krun_create_ctx")?;
+    // 2 vCPU / 1024 MiB: 512 MiB is the practical floor to reach Alpine
+    // userspace; 1024 MiB is cheap headroom under Hypervisor.framework with no
+    // boot penalty. (Stay below the kernel's CONFIG_NR_CPUS.) The boot command
+    // may expose these as flags in a future change.
+    let cfg = VmConfig::new(2, 1024, kernel, rootfs);
+    cfg.apply(&mut ctx)
+        .context("applying VmConfig to krun context")?;
+
+    // Set the guest workload. libkrun's own init (`/init.krun`) runs as PID 1,
+    // mounts /proc,/sys,/dev, reads this target from /.krun_config.json, and
+    // forks it — so the rootfs needs no init system, only this binary. Default
+    // to the v0.1 bring-up stub; `MINVMD_EXEC` overrides (e.g. /sbin/minimald).
+    // Pass an explicit minimal envp (not None): None inherits the full host env
+    // into the guest, which can overflow the ~2 KiB aarch64 kernel cmdline.
+    let exec =
+        std::env::var("MINVMD_EXEC").unwrap_or_else(|_| "/sbin/minvmd-stub-init".to_string());
+    let argv: [&str; 0] = [];
+    ctx.set_exec(&exec, &argv, Some(&["PATH=/usr/sbin:/usr/bin:/sbin:/bin"]))
+        .with_context(|| format!("setting guest exec workload: {exec}"))?;
+
+    // Optional early-boot console capture for diagnosing a stuck boot. Off by
+    // default; set `MINVMD_BOOT_LOG=<path>` to capture hvc0 to a host file.
+    if let Some(log_path) = std::env::var_os("MINVMD_BOOT_LOG") {
+        ctx.set_console_output(&log_path)
+            .context("setting console output log")?;
+    }
+
+    // Register the READY-marker vsock port (guest→host). The plain
+    // `krun_add_vsock_port` is `krun_add_vsock_port2(.., listen=false)`: the
+    // parent listens on the host UDS `marker_sock`; the guest workload connects
+    // to AF_VSOCK CID 2 (host) port VSOCK_MARKER_PORT and writes `READY\n`,
+    // which libkrun bridges to the parent (R2.4).
+    ctx.add_vsock_port(VSOCK_MARKER_PORT, &marker_sock)
+        .context("registering READY-marker vsock port")?;
+
+    tracing::info!(
+        port = VSOCK_MARKER_PORT,
+        sock = %marker_sock,
+        exec = %exec,
+        "guest workload + READY-marker vsock port set; calling krun_start_enter"
+    );
+
+    // start_enter consumes the context and boots the VM. On success, libkrun
+    // exit()s the process with the guest workload's exit code and never
+    // returns. On failure it returns a VmError; we surface it as an anyhow
+    // error so the parent can observe the child's non-zero exit.
+    let err = ctx.start_enter();
+    bail!("krun_start_enter returned unexpectedly: {err}");
+}