norrietaylor · Jun 9, 2026 · Jun 7, 2026 · Jun 7, 2026 · Jun 7, 2026 · Jun 7, 2026
@@ -38,38 +38,54 @@ permissions:
   contents: read
 
 jobs:
-  virtio-kernel:
-    # Pull the prebuilt virtio-linux kernel (vmlinuz / Image.gz) from the public
-    # minimal build cache — no kernel build, no GCS auth — and hand it to the
-    # macOS boot job. The packaging `minimal` CLI is Linux-only, so this runs on
-    # Linux. Cheap (a cache pull), so it is unconditional and decoupled from the
-    # self-hosted runner.
+  artifacts:
+    # Build the `minimal` CLI from THIS repo's sources, then materialize the
+    # guest kernel + rootfs against the repo's own .minimal/minimal.toml. Both
+    # are cache pulls keyed by the pinned upstream commit (not compiles): the
+    # upstream virtio-kernel-raw / microvm-rootfs packages' prebuilt aarch64
+    # artifacts are in the public cache, so they pull fine on a cheap x86_64
+    # Linux runner — no native arm64 build and no self-hosted runner needed.
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v6
         with:
           persist-credentials: false
-      - name: Fetch virtio-linux kernel from the build cache
-        run: ./scripts/fetch-virtio-kernel.sh "$RUNNER_TEMP/vmlinuz" aarch64
+      - name: Toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install protoc
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Cache cargo registry and build artifacts
+        uses: Swatinem/rust-cache@v2
+      - name: Materialize guest kernel (raw Image)
+        run: ./scripts/fetch-artifact.sh virtio-kernel "$RUNNER_TEMP/vmlinuz" aarch64
+      - name: Materialize guest rootfs (ext4 image)
+        run: ./scripts/fetch-artifact.sh minvmd-rootfs "$RUNNER_TEMP/rootfs.img" aarch64
       - name: Upload kernel artifact
         uses: actions/upload-artifact@v4
         with:
           name: virtio-kernel-aarch64
           path: ${{ runner.temp }}/vmlinuz
           if-no-files-found: error
+      - name: Upload rootfs artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: minvmd-rootfs-aarch64
+          path: ${{ runner.temp }}/rootfs.img
+          if-no-files-found: error
 
   boot-e2e:
     # Boot E2E (R2.4 READY round-trip) on real hardware, using the cache-pulled
-    # kernel and the real minvmd guest rootfs (Alpine + READY-writer stub).
-    # Separate from build-macos so a kernel-fetch hiccup never blocks the
+    # kernel + rootfs ext4 image (both from the `artifacts` job).
+    # Separate from build-macos so an artifact-fetch hiccup never blocks the
     # always-green clippy/test/smoke checks.
     #
-    # GATING: green on the runner — kernel PE_GZ load, virtio-fs root,
-    # /init.krun exec of the stub, and the guest READY connect-out on vsock 7350
-    # (krun_add_vsock_port == listen=false; host + guest both 7350). See #326.
+    # GATING: green on the runner — raw kernel load (KRUN_KERNEL_FORMAT_RAW),
+    # ext4 block root (/dev/vda via krun_add_disk2), kernel `init=` exec of the
+    # stub (no /init.krun on a block root), and the guest READY connect-out on
+    # vsock 7350 (krun_add_vsock_port == listen=false; host + guest both 7350).
     if: ${{ vars.RUN_MACOS_CI != 'false' }}
-    needs: virtio-kernel
+    needs: [artifacts]
     runs-on: [self-hosted, macOS, ARM64]
     timeout-minutes: 20
     steps:
@@ -89,12 +105,11 @@ jobs:
         with:
           name: virtio-kernel-aarch64
           path: ${{ runner.temp }}/kernel
-      - name: Stage minvmd guest rootfs
-        # Real R2.2 rootfs: fetch the pinned Alpine minirootfs and overlay socat
-        # + the READY-writer stub (/sbin/minvmd-stub-init) via build-rootfs.sh.
-        # A setup failure fails the job — it is not hidden behind the non-gating
-        # test below.
-        run: scripts/build-rootfs.sh
+      - name: Download minvmd-rootfs image
+        uses: actions/download-artifact@v4
+        with:
+          name: minvmd-rootfs-aarch64
+          path: ${{ runner.temp }}/rootfs
       - name: Build boot + bridge E2E + minvmd (no run)
         # Build both test harnesses + the minvmd bin, then codesign, then run the
         # test BINARIES DIRECTLY (below). Invoking `cargo test` again after
@@ -105,15 +120,16 @@ jobs:
       - name: Codesign minvmd (hypervisor entitlement, R1.4)
         run: codesign --entitlements crates/minvmd/minvmd.entitlements --force -s - target/debug/minvmd
       - name: Boot E2E (READY round-trip)
-        # Gating: a real Alpine microVM must boot and the guest must write READY
-        # over vsock 7350 within the budget. Also validates the virtio-linux
-        # kernel config (virtio-MMIO / VIRTIO_FS / VSOCK / HVC).
+        # Gating: a real microVM must boot from the ext4 root image and the
+        # guest must write READY over vsock 7350 within the budget. Also
+        # validates the virtio-linux kernel config (virtio-MMIO / VIRTIO_BLK /
+        # EXT4 / VSOCK / HVC).
         run: |
           testbin="$(ls -1t target/debug/deps/boot_e2e-* | grep -v '\.d$' | head -1)"
           test -x "$testbin" || { echo "::error::boot_e2e test binary not found"; exit 1; }
           MINVMD_E2E=1 \
           MINVMD_KERNEL_PATH="$RUNNER_TEMP/kernel/vmlinuz" \
-          MINVMD_ROOTFS_PATH="$(scripts/build-rootfs.sh --print-path)" \
+          MINVMD_ROOTFS_PATH="$RUNNER_TEMP/rootfs/rootfs.img" \
           MINVMD_BOOT_LOG="$RUNNER_TEMP/boot.log" \
             "$testbin" --include-ignored --nocapture
       - name: Bridge E2E (concurrent UDS↔vsock round-trip)
@@ -127,8 +143,16 @@ jobs:
           test -x "$testbin" || { echo "::error::bridge_e2e test binary not found"; exit 1; }
           MINVMD_E2E=1 \
           MINVMD_KERNEL_PATH="$RUNNER_TEMP/kernel/vmlinuz" \
-          MINVMD_ROOTFS_PATH="$(scripts/build-rootfs.sh --print-path)" \
+          MINVMD_ROOTFS_PATH="$RUNNER_TEMP/rootfs/rootfs.img" \
             "$testbin" --include-ignored --nocapture
+      - name: Boot latency benchmark (informational)
+        # Report boot-to-READY min/median/max on the runner. Uses the already
+        # codesigned minvmd; non-gating (boot correctness is gated by the e2e
+        # tests above), so a timing hiccup never reds the build.
+        run: |
+          MINVMD_KERNEL_PATH="$RUNNER_TEMP/kernel/vmlinuz" \
+          MINVMD_ROOTFS_PATH="$RUNNER_TEMP/rootfs/rootfs.img" \
+            scripts/bench-minvmd-boot.sh 10 target/debug/minvmd || true
       - name: Upload guest boot console log
         # Capture hvc0 output for diagnosing a stuck/failed boot (e.g. a missing
         # virtio driver in the kernel). Always runs so a red boot is debuggable.
@@ -147,7 +171,7 @@ jobs:
     # minimal2 -> `minvmd run --detach` -> VM boot path end to end, the macOS
     # integration gate that no unit test or Linux CI can cover.
     if: ${{ vars.RUN_MACOS_CI != 'false' }}
-    needs: virtio-kernel
+    needs: [artifacts]
     runs-on: [self-hosted, macOS, ARM64]
     timeout-minutes: 20
     steps:
@@ -167,8 +191,11 @@ jobs:
         with:
           name: virtio-kernel-aarch64
           path: ${{ runner.temp }}/kernel
-      - name: Stage minvmd guest rootfs
-        run: scripts/build-rootfs.sh
+      - name: Download minvmd-rootfs image
+        uses: actions/download-artifact@v4
+        with:
+          name: minvmd-rootfs-aarch64
+          path: ${{ runner.temp }}/rootfs
       - name: Build minvmd + minimal2 (no run)
         # Build both bins before the single codesign. minimal2 spawns `minvmd`
         # by name from PATH; only minvmd needs the hypervisor entitlement, so
@@ -186,7 +213,7 @@ jobs:
           set -uo pipefail   # not -e: capture failures so we can dump diagnostics
           export PATH="$PWD/target/debug:$PATH"
           export MINVMD_KERNEL_PATH="$RUNNER_TEMP/kernel/vmlinuz"
-          export MINVMD_ROOTFS_PATH="$(scripts/build-rootfs.sh --print-path)"
+          export MINVMD_ROOTFS_PATH="$RUNNER_TEMP/rootfs/rootfs.img"
           # Capture the guest console; this env var propagates through the
           # minimal2 -> `minvmd run --detach` -> `minvmd run` -> __krun-vmm chain,
           # so the silent detached supervisor's boot is still observable here.

@@ -1,3 +1,6 @@
 target/
 # Serena MCP working-tree state (spectacles quick-setup).
 .serena/
+
+# minvmd local dev scratch (kernel/rootfs materialized for local boot)
+.scratch/
@@ -1,7 +1,7 @@
 [upstream] # Source of software & tooling
 repo = "https://github.com/gominimal/pkgs"
 branch = "main"
-locked_commit = "0da02c7fe4cff292388c1b9ebd611feceabb22fe"
+locked_commit = "64e6866"
 
 [harness]
 use = "rust"
@@ -42,6 +42,19 @@ packages = ["bash", "coreutils", "diffutils", "findutils", "tar"]
 env_vars.test = "ello"
 cmd = ["/usr/bin/bash", "-l"]
 
+# Raw (uncompressed) kernel Image from the upstream virtio-kernel-raw package,
+# loaded via KRUN_KERNEL_FORMAT_RAW (no in-VMM gzip decompress) for faster boot.
+[outputs.virtio-kernel]
+type = "raw-file"
+packages = ["virtio-kernel-raw"]
+path = "usr/share/virtio-linux/Image&q
8
uot;
+
+# Guest rootfs ext4 image from the upstream microvm-rootfs package.
+[outputs.minvmd-rootfs]
+type = "raw-file"
+packages = ["microvm-rootfs"]
+path = "usr/share/microvm-rootfs/rootfs.img"
+
 [tasks.vim]
 profile = "demo"
 state_key = ""

@@ -0,0 +1,109 @@
+# minvmd
+
+macOS-only host daemon that boots a Linux microVM via libkrun and bridges a host
+UNIX socket to in-VM `minimald`, so the `minimal` CLI reaches a Linux session
+daemon on macOS without knowing a VM exists. Spec:
+`docs/specs/01-spec-minvmd-host-daemon/`.
+
+The guest boots from two artifacts produced by the `minimal` package system,
+both from upstream `gominimal/pkgs`:
+- **kernel** — the `virtio-kernel-raw` package's uncompressed `Image`
+  (`virtio-kernel` output).
+- **rootfs** — the `microvm-rootfs` package's ext4 image (`minvmd-rootfs`
+  output), loaded as a block device (`krun_add_disk2` → `/dev/vda`).
+
+## Run the boot verification locally (Apple Silicon)
+
+Prereqs:
+- Apple Silicon Mac.
+- `brew install slp/krun/libkrun` — third-party tap; required by minvmd.
+- The `minimal` shim on `PATH` (`~/.minimal/shim/bin/minimal`). On macOS,
+  `materialize` runs `minimal` inside a Linux VM; a from-source macOS build
+  cannot run the build pipeline (sandbox2 is Linux-only).
+
+```sh
+# 1. Materialize the kernel + guest rootfs INTO THE REPO.
+#    macOS caveat: the shim runs minimal in a VM and only syncs the project dir
+#    back to the host, so --output MUST be a path under the repo. A /tmp path is
+#    written inside the VM and never appears on the host.
+#    The kernel output is an uncompressed Image (virtio-kernel-raw decompresses
+#    upstream's Image.gz at build time), loaded raw (KRUN_KERNEL_FORMAT_RAW) to
+#    skip libkrun's ~77 ms in-VMM decompress.
+mkdir -p .scratch
+minimal materialize --output .scratch/vmlinuz    --arch aarch64 virtio-kernel
+minimal materialize --output .scratch/rootfs.img --arch aarch64 minvmd-rootfs
+
+# 2. Build minvmd (+ minimal2 for the autospawn path) WITHOUT running.
+cargo build -p minvmd --bin minvmd -p minimal2 --bin minimal2
+
+# 3. Codesign minvmd with the hypervisor entitlement. This MUST be the last
+#    thing to touch the binary: a later `cargo run`/`cargo test` relinks and
+#    unsigns it, and krun_start_enter then fails with EINVAL.
+codesign --entitlements crates/minvmd/minvmd.entitlements --force -s - target/debug/minvmd
+
+# 4. Run. MINVMD_ROOTFS_PATH is the ext4 .img FILE (block device), not a dir.
+export PATH="$PWD/target/debug:$PATH"
+export MINVMD_KERNEL_PATH="$PWD/.scratch/vmlinuz"
+export MINVMD_ROOTFS_PATH="$PWD/.scratch/rootfs.img"
+minimal2 ls        # cold: auto-spawns minvmd, boots the VM, prints []
+minvmd status      # running
+minimal2 ls        # warm: < 500 ms
+minvmd stop
+```
+
+## E2E tests (boot + bridge)
+
+`MINVMD_E2E=1`-gated, `#[ignore]` by default. Run the prebuilt test binary
+directly — `cargo test` after signing relinks and unsigns `minvmd`.
+
+```sh
+cargo test -p minvmd --test boot_e2e --test bridge_e2e --no-run
+codesign --entitlements crates/minvmd/minvmd.entitlements --force -s - target/debug/minvmd
+testbin="$(ls -1t target/debug/deps/boot_e2e-* | grep -v '\.d$' | head -1)"
+MINVMD_E2E=1 \
+MINVMD_KERNEL_PATH="$PWD/.scratch/vmlinuz" \
+MINVMD_ROOTFS_PATH="$PWD/.scratch/rootfs.img" \
+  "$testbin" --include-ignored --nocapture
+```
+
+## Boot-latency benchmark
+
+`scripts/bench-minvmd-boot.sh` times `minvmd boot` to the guest READY marker over
+N runs (default 10) and reports min/median/max. Needs a codesigned `minvmd` and
+the kernel + rootfs paths:
+
+```sh
+MINVMD_KERNEL_PATH="$PWD/.scratch/vmlinuz" \
+MINVMD_ROOTFS_PATH="$PWD/.scratch/rootfs.img" \
+  scripts/bench-minvmd-boot.sh
+```
+
+CI runs it (informational) in the `boot-e2e` job. Typical: ~113 ms median on
+the CI runner (Apple Silicon; faster on less-loaded hardware) — see the
+uncompressed-kernel note below.
+
+## How it boots
+
+- Kernel loaded via `krun_set_kernel` as a raw uncompressed aarch64 `Image`
+  (`KRUN_KERNEL_FORMAT_RAW`). The `virtio-kernel` output is built by the upstream
+  `virtio-kernel-raw` package, which decompresses upstream's `Image.gz` at build
+  time. Loading raw skips libkrun's in-VMM gzip decompress (~77 ms, over half of
+  boot-to-READY).
+- Rootfs loaded via `krun_add_disk2` as `/dev/vda`; cmdline
+  `console=hvc0 root=/dev/vda rootfstype=ext4 ro init=<exec-target>`. A block
+  root has **no** libkrun `/init.krun`, so the kernel runs the exec target
+  (`MINVMD_EXEC`, default `/sbin/microvm-init`) directly as PID 1; devtmpfs
+  auto-mounts `/dev`, giving the workload `/dev/vsock`.
+- The guest writes `READY\n` on vsock port 7350 (boot marker, R2.4) and serves
+  the bridge on vsock port 2222 (R3); `minvmd` registers the host UDS via
+  `krun_add_vsock_port2(.., listen = true)`.
+
+## Notes
+
+- The guest rootfs is the upstream `microvm-rootfs` package (an ext4 image built
+  from `base` + `socat`, packed with `mke2fs`). The exec target is the bring-up
+  stub `/sbin/microvm-init`; `/sbin/minimald` is the production target (Stage 2).
+- In CI (`.github/workflows/ci-macos.yml`) the kernel + rootfs are materialized
+  on a cheap Linux runner (`scripts/fetch-artifact.sh` — cache pulls of the
+  upstream packages' prebuilt aarch64 artifacts) and handed to the self-hosted
+  boot jobs as artifacts.
@@ -13,7 +13,7 @@ pub mod vmm_child;
 /// the VMM child. The parent reads `READY` to confirm boot (R2.4).
 ///
 /// Must match the guest rootfs's READY/agent port. The canonical value is
-/// `7350` — both the guest rootfs manifest (`etc/minvmd/manifest`:
+/// `7350` — both the guest rootfs manifest (`etc/microvm/manifest`:
 /// `vsock_port_ready=7350`) and the reference impl (min-ctl: cold boot to a TCP
 /// connection on `127.0.0.1:7350`) use it. The host previously used `9799`,
 /// which no guest emits on, so the READY marker never arrived.

@@ -7,9 +7,11 @@
 //! On macOS:
 //! 1. Reads kernel and rootfs from `MINVMD_KERNEL_PATH` / `MINVMD_ROOTFS_PATH`.
 //! 2. Reads the READY-marker socket path from `MINVMD_MARKER_SOCK`.
-//! 3. Creates and configures a libkrun context (kernel, rootfs, resources).
-//! 4. Sets the guest workload via `krun_set_exec` (libkrun's `/init.krun` runs
-//!    as PID 1 and forks it); default `/sbin/minvmd-stub-init`, `MINVMD_EXEC`
+//! 3. Creates and configures a libkrun context (kernel + cmdline, ext4 root
+//!    disk via `krun_add_disk2`, resources).
+//! 4. Sets the guest workload via the kernel `init=` cmdline: the ext4 block
+//!    root has no libkrun `/init.krun`, so the kernel execs the workload
+//!    directly as PID 1. Default `/sbin/microvm-init`, `MINVMD_EXEC`
 //!    overrides.
 //! 5. Registers the marker socket for `VSOCK_MARKER_PORT` (guest→host): the
 //!    guest workload connects to that vsock port and writes `READY\n`, which
@@ -45,27 +47,22 @@ fn run_macos() -> Result<()> {
         format!("reading {MARKER_SOCK_ENV}: VMM child must be spawned by `minvmd boot`")
     })?;
 
+    // Guest workload run as the kernel `init=`. The root is a block device
+    // (ext4 via krun_add_disk2), which has no libkrun `/init.krun`, so the
+    // kernel execs this directly as PID 1. Default to the v0.1 bring-up stub;
+    // `MINVMD_EXEC` overrides (e.g. /sbin/minimald in Stage 2).
+    let exec = std::env::var("MINVMD_EXEC").unwrap_or_else(|_| "/sbin/microvm-init".to_string());
+
     let mut ctx = Context::create().context("krun_create_ctx")?;
-    // 2 vCPU / 1024 MiB: 512 MiB is the practical floor to reach Alpine
-    // userspace; 1024 MiB is cheap headroom under Hypervisor.framework with no
-    // boot penalty. (Stay below the kernel's CONFIG_NR_CPUS.) The boot command
-    // may expose these as flags in a future change.
-    let cfg = VmConfig::new(2, 1024, kernel, rootfs);
+    // 2 vCPU / 1024 MiB: 512 MiB is the practical floor to reach userspace;
+    // 1024 MiB is cheap headroom under Hypervisor.framework with no boot
+    // penalty. (Stay below the kernel's CONFIG_NR_CPUS.) The boot command may
+    // expose these as flags in a future change. `apply` configures the kernel +
+    // cmdline (with `init=exec`), the ext4 root disk, and the vsock bridge.
+    let cfg = VmConfig::new(2, 1024, kernel, rootfs, exec.clone());
     cfg.apply(&mut ctx)
         .context("applying VmConfig to krun context")?;
 
-    // Set the guest workload. libkrun's own init (`/init.krun`) runs as PID 1,
-    // mounts /proc,/sys,/dev, reads this target from /.krun_config.json, and
-    // forks it — so the rootfs needs no init system, only this binary. Default
-    // to the v0.1 bring-up stub; `MINVMD_EXEC` overrides (e.g. /sbin/minimald).
-    // Pass an explicit minimal envp (not None): None inherits the full host env
-    // into the guest, which can overflow the ~2 KiB aarch64 kernel cmdline.
-    let exec =
-        std::env::var("MINVMD_EXEC").unwrap_or_else(|_| "/sbin/minvmd-stub-init".to_string());
-    let argv: [&str; 0] = [];
-    ctx.set_exec(&exec, &argv, Some(&["PATH=/usr/sbin:/usr/bin:/sbin:/bin"]))
-        .with_context(|| format!("setting guest exec workload: {exec}"))?;
-
     // Optional early-boot console capture for diagnosing a stuck boot. Off by
     // default; set `MINVMD_BOOT_LOG=<path>` to capture hvc0 to a host file.
     if let Some(log_path) = std::env::var_os("MINVMD_BOOT_LOG") {