norrietaylor · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026
 @@ -0,0 +1,34 @@
+# Nextest configuration. CI lanes select a profile explicitly (--profile ci);
+# local `cargo nextest run` uses profile.default.
+#
+# No retries anywhere: there is no recorded flakiness in the workspace suite,
+# and for VM boot tests a same-invocation retry runs against the leaked
+# __krun-vmm/gvproxy processes of the failed attempt (#588) — the lane reap
+# steps plus slow-timeout terminate-after are the mitigation. Revisit once the
+# harness reaps its own process group.
+
+[profile.default]
+# Local runs: surface slow tests, never kill them.
+slow-timeout = "60s"
+
+[profile.ci]
+# Workspace suite (core-tests composite). A per-test override with a larger
+# slow-timeout is the escape hatch for a legitimately slow test — not raising
+# the cap.
+fail-fast = false
+retries = 0
+slow-timeout = { period = "60s", terminate-after = 5 }
+leak-timeout = "1s"
+final-status-level = "slow"
+
+[profile.vm]
+# Prebuilt-harness VM runs (the KVM lane's archived e2e tests). One VM at a
+# time on a 2-vCPU runner; boots are legitimately slow, so the slow period is
+# generous — but a hung boot dies at 6 minutes instead of burning the job
+# timeout. The vmm/gvproxy children linger briefly at teardown by design,
+# hence the longer leak window.
+test-threads = 1
+retries = 0
+fail-fast = true
+slow-timeout = { period = "120s", terminate-after = 3 }
+leak-timeout = "5s"
@@ -19,7 +19,9 @@ runs:
       run: cargo fetch --locked
     - name: Run tests
       shell: bash
-      run: cargo nextest run --workspace
+      # --no-tests=fail: an empty selection is a broken filter or a dropped
+      # target, not a pass.
+      run: cargo nextest run --workspace --profile ci --no-tests=fail
     - name: Run doctests
       shell: bash
       run: cargo test --workspace --doc
@@ -37,3 +37,16 @@ runs:
           echo "LIBKRUN_PREFIX=$prefix"
           echo "LD_LIBRARY_PATH=$prefix${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
         } >> "$GITHUB_ENV"
+    - name: Verify libkrun exports krun_add_disk3 (>= 1.19.0)
+      # Early, actionable failure if the materialized prefix is too old for
+      # krun_add_disk3 (the /dev/vdb attach) — at the point every consumer of
+      # this composite links it, instead of a cryptic runtime error at VM
+      # boot. minvmd's compile-time link is the real backstop; this mirrors
+      # the export check the macOS composite performs on its built dylib.
+      shell: bash
+          run: |
+        so="$(ls "$HOME/.krun"/libkrun.so* 2>/dev/null | head -1)"
+        test -n "$so" || { echo "::error::no libkrun.so under $HOME/.krun"; exit 1; }
+        nm -D "$so" 2>/dev/null | grep -q krun_add_disk3 \
+          || { echo "::error::materialized libkrun ($so) does not export krun_add_disk3; need >= 1.19.0"; exit 1; }
+        echo "libkrun OK: $so exports krun_add_disk3"
@@ -1,6 +1,6 @@
 name: CI (Linux KVM)
 
-# minvmd Linux/KVM end-to-end on a GitHub-hosted runner (#410, #411).
+# minvmd Linux/KVM end-to-end on a GitHub-hosted runner.
 #
 # It needs no preprovisioned runner — KVM is available on GitHub-hosted
 # x86_64 Linux runners, and libkrun is materialized from the upstream package
@@ -11,7 +11,7 @@ name: CI (Linux KVM)
 # /dev/kvm, so the Linux/KVM e2e can only run on x86_64. aarch64 + libkrun boot
 # is already covered by the macOS lane (ci-macos.yml, Apple Silicon / HVF).
 #
-# GATE-READY (#687): ALWAYS-TRIGGERED, path-scoped INSIDE the workflow — the
+# GATE-READY: ALWAYS-TRIGGERED, path-scoped INSIDE the workflow — the
 # `changes` job gates the e2e, and the `if: always()` aggregator
 # (ci-linux-kvm-success) reports success even when it was path-skipped, so
 # the context can join the required checks after this lane soaks green. A
@@ -110,10 +110,10 @@ jobs:
             'apt-get update && apt-get install -y --no-install-recommends \
                protobuf-compiler libprotobuf-dev cpio git'
 
-      - name: Install cross
+      - name: Install cross + nextest
         uses: taiki-e/install-action@v2
         with:
-          tool: cross
+          tool: cross,nextest
 
       - name: Cache cargo + build artifacts
         uses: actions/cache@v6
@@ -143,24 +143,28 @@ jobs:
       - name: Build minvmd
         run: cargo build -p minvmd --bin minvmd
 
-      - name: Build e2e harnesses + stage the testbed
-        # The manifest records exactly which hash-suffixed binaries cargo
-        # produced; the test job resolves them via scripts/ci/run-testbin.sh
-        # (never `ls | head`). The harnesses locate minvmd via the MINVMD_BIN
-        # override (their compile-time CARGO_BIN_EXE path does not exist on
-        # the test runner).
+      - name: Build the minimal CLI
+        # SEPARATE invocation from the minvmd build: a combined `cargo build
+        # -p minvmd -p minimal` unifies minvmd's default `libkrun` feature
+        # into the CLI's default-features=false opt-out, linking libkrun into
+        # `minimal` (the regression the macOS lane hit and now guards). The
+        # CLI drives the unified session e2e in the test job.
+        run: cargo build -p minimal --bin minimal
+
+      - name: Archive e2e harnesses + stage the testbed
+        # `cargo nextest archive` replaces the hand-rolled testbins.json
+        # manifest: the archive carries every minvmd test binary with exec
+        # bits intact, and the test job selects harnesses with filtersets.
+        # The sidecar files stay explicit — libkrun lives outside target/ (an
+        # archive cannot carry it), and the harnesses/CLI locate minvmd via
+        # MINVMD_BIN / PATH (their compile-time CARGO_BIN_EXE path does not
+        # exist on the test runner).
         run: |
           set -euo pipefail
           mkdir -p testbed
-          cargo test -p minvmd --test boot_e2e --test minimald_session_e2e --test vsock_relay_e2e \
-            --no-run --message-format=json > "$RUNNER_TEMP/build-plan.json"
-          jq -rs '[.[] | select(.reason=="compiler-artifact" and .executable != null and (.target.kind | index("test")))
-                   | {key: .target.name, value: (.executable | split("/") | last)}] | from_entries' \
-            "$RUNNER_TEMP/build-plan.json" > testbed/testbins.json
-          jq -r 'select(.reason=="compiler-artifact" and .executable != null and (.target.kind | index("test"))) | .executable' \
-            "$RUNNER_TEMP/build-plan.json" | while read -r exe; do cp "$exe" testbed/; done
-          cat testbed/testbins.json
+          cargo nextest archive -p minvmd --archive-file testbed/nextest-archive.tar.zst
           cp target/debug/minvmd testbed/minvmd
+          cp target/debug/minimal testbed/minimal
           cp "$RUNNER_TEMP/initramfs.cpio" testbed/initramfs.cpio
           # Ship the exact libkrun prefix the binaries were linked against.
           tar czf testbed/libkrun-prefix.tgz -C "$LIBKRUN_PREFIX" .
@@ -174,10 +178,10 @@ jobs:
           retention-days: 3
 
   test-kvm:
-    # TEST phase: no Rust toolchain, no cargo — download the testbed, pull
-    # the guest images, and run the prebuilt harnesses directly. A flake
-    # rerun of this job alone costs ~15 minutes instead of the old
-    # 60-minute build+test monolith.
+    # TEST phase: no Rust toolchain, no cargo build — download the testbed,
+    # pull the guest images, and run the archived harnesses with the
+    # standalone cargo-nextest binary. A flake rerun of this job alone costs
+    # ~15 minutes instead of the old 60-minute build+test monolith.
     needs: [changes, build-linux]
     if: github.event_name == 'workflow_dispatch' || needs.changes.outputs.kvm == 'true'
     runs-on: ubuntu-latest # x86_64; KVM-capable
@@ -187,6 +191,11 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Install nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: nextest
+
       - name: Enable /dev/kvm access
         # GitHub-hosted x86_64 Linux runners expose /dev/kvm but the runner user
         # is not in the `kvm` group; widen the node so libkrun can open it. Fail
@@ -209,18 +218,23 @@ jobs:
           path: ${{ runner.temp }}/testbed
 
       - name: Unpack testbed + resolve paths
-        # Artifact transfer strips exec bits; restore them from the manifest.
-        # MINVMD_BIN points the harnesses at the shipped minvmd (their baked
-        # CARGO_BIN_EXE path only exists on the build runner). The libkrun
-        # prefix is unpacked to the same $HOME/.krun path the build linked
        # (rpath) and exported for the dynamic loader.
+        # Artifact transfer strips exec bits on the sidecar binaries; restore
+        # them (the nextest archive preserves its own). MINVMD_BIN points the
+        # harnesses at the shipped minvmd (their baked CARGO_BIN_EXE path only
+        # exists on the build runner); the testbed joins PATH so the CLI's
+        # autospawn and the lifecycle script resolve minvmd/minimal by name.
+        # The libkrun prefix is unpacked to the same $HOME/.krun path the
+        # build linked (rpath) and exported for the dynamic loader; its
+        # krun_add_disk3 export was verified at build time by the
+        # setup-libkrun-linux composite — minvmd loading it here is the
+        # runtime backstop.
         run: |
           set -euo pipefail
           TESTBED="$RUNNER_TEMP/testbed"
-          chmod +x "$TESTBED/minvmd"
-          jq -r '.[]' "$TESTBED/testbins.json" | while read -r b; do chmod +x "$TESTBED/$b"; done
+          chmod +x "$TESTBED/minvmd" "$TESTBED/minimal"
           mkdir -p "$HOME/.krun"
           tar xzf "$TESTBED/libkrun-prefix.tgz" -C "$HOME/.krun"
+          echo "$TESTBED" >> "$GITHUB_PATH"
           {
             echo "TESTBED=$TESTBED"
             echo "MINVMD_BIN=$TESTBED/minvmd"
@@ -232,19 +246,8 @@ jobs:
             echo "MINVMD_INITRAMFS=$TESTBED/initramfs.cpio"
           } >> "$GITHUB_ENV"
 
-      - name: Assert libkrun exports krun_add_disk3 (>= 1.19.0)
-        # krun_add_disk3 (the /dev/vdb attach) needs libkrun >= 1.19.0. Fail
-        # loudly here rather than with a cryptic runtime error at VM boot if
-        # the shipped upstream libkrun is too old.
-        run: |
-          so="$(ls "$LIBKRUN_PREFIX"/libkrun.so* 2>/dev/null | head -1)"
-          test -n "$so" || { echo "::error::no libkrun.so under $LIBKRUN_PREFIX"; exit 1; }
-          nm -D "$so" 2>/dev/null | grep -q krun_add_disk3 \
-            || { echo "::error::shipped libkrun ($so) does not export krun_add_disk3; need >= 1.19.0"; exit 1; }
-          echo "libkrun OK: $so exports krun_add_disk3"
-
-      - name: Materialize gvproxy switch binary (DM1 relay)
-        # The DM1 gvproxy relay e2e (vsock_relay_e2e) spawns this per-host switch
+      - name: Materialize gvproxy switch binary
+        # The gvproxy relay e2e (vsock_relay_e2e) spawns this per-host switch
         # binary; pin-verified, no Go toolchain. Publish its path for the gated
         # step below.
         run: |
@@ -267,15 +270,22 @@ jobs:
           arch: x86_64
           channel: unstable
 
-      - name: Boot E2E (READY round-trip, R3.1)
+      - name: Boot E2E (READY round-trip)
         env:
           MINVMD_BOOT_LOG: ${{ runner.temp }}/boot.log
-        run: ./scripts/ci/run-testbin.sh "$TESTBED" boot_e2e --include-ignored --nocapture
+        run: |
+          cargo-nextest nextest run --archive-file "$TESTBED/nextest-archive.tar.zst" \
+            --workspace-remap "$GITHUB_WORKSPACE" --profile vm \
+            --run-ignored all --no-tests=fail -E 'binary(boot_e2e)'
 
       - name: Session E2E (full minimald session over the bridge)
-        run: ./scripts/ci/run-testbin.sh "$TESTBED" minimald_session_e2e --include-ignored --nocapture --exact minimald_exec_over_bridge
+        run: |
+          cargo-nextest nextest run --archive-file "$TESTBED/nextest-archive.tar.zst" \
+            --workspace-remap "$GITHUB_WORKSPACE" --profile vm \
+            --run-ignored all --no-tests=fail \
+            -E 'binary(minimald_session_e2e) and test(=minimald_exec_over_bridge)'
 
-      - name: DM1 gvproxy relay E2E (OwnIp PTask gets a switch IP, R1.5)
+      - name: Gvproxy relay E2E (OwnIp PTask gets a switch IP)
         # Boots an OwnIp VM and asserts its netns receives a 100.64.0.0/16 switch
         # IP via the async TAP↔gvproxy relay, with frames traversing the switch.
         # Needs /dev/kvm, the gvproxy binary, and CAP_NET_ADMIN (tap + netns), so
@@ -286,7 +296,9 @@ jobs:
           MINVMD_INTEGRATION_TEST: "1"
         run: |
           sudo -E env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" \
-            ./scripts/ci/run-testbin.sh "$TESTBED" vsock_relay_e2e --include-ignored --nocapture
+            cargo-nextest nextest run --archive-file "$TESTBED/nextest-archive.tar.zst" \
+              --workspace-remap "$GITHUB_WORKSPACE" --profile vm \
+              --run-ignored all --no-tests=fail -E 'binary(vsock_relay_e2e)'
 
       # NOTE: bridge_e2e is intentionally NOT run here. It targets the Stage-1
       # guest "vsock stub" (a socat/cat echo on port 2222) that minimald-as-pid1
@@ -295,53 +307,39 @@ jobs:
       # minimald_session_e2e; session coverage of the bridge is provided by
       # Session E2E above plus the daemon lifecycle step below.
 
-      - name: Daemon lifecycle (run --detach → status → stop, R4.2/R4.3/R4.4)
-        # Exercises the supervised daemon path that manages the lifecycle state,
-        # which the raw `boot` e2e tests above do not cover.
+      - name: Reap leftover VM processes
+        # The harness runs above can leak __krun-vmm / gvproxy children on
+        # failure (Guest::drop kills only its own minvmd), and a leftover
+        # wedges the vsock bridge for the fresh daemons the next two steps
+        # spawn (lifecycle, then the CLI session e2e). Root-owned leftovers
+        # are possible here (the relay e2e runs under sudo), hence sudo.
+        # Ported from the macOS lane; the proper fix is for the harness to
+        # reap its own process group, tracked separately.
+        run: |
+          sudo pkill -x minvmd || true
+          sudo pkill -f __krun-vmm || true
+          sudo pkill -f gvproxy || true
+
+      - name: Daemon lifecycle (run --detach → status → stop)
+        # Exercises the supervised daemon path that manages the lifecycle
+        # state, which the harness e2e tests above do not cover. The script
+        # resolves minvmd from PATH (the testbed dir, added at unpack).
         env:
           MINVMD_BOOT_LOG: ${{ runner.temp }}/boot-daemon.log
-        run: |
-          set -euo pipefail
-          BIN="$MINVMD_BIN"
-
-          echo "::group::run --detach"
-          "$BIN" run --detach --timeout 30
-          echo "::endgroup::"
-
-          echo "::group::status (expect running)"
-          # `run --detach` returns once the host UDS accepts connections, which
-          # libkrun opens early in VM setup — slightly ahead of the supervisor's
-          # Starting->Running transition (set after the guest READY marker). Poll
-          # status (up to ~15s) for Running rather than asserting immediately.
-          # jq parses structurally (preinstalled on GitHub runners) so the check
-          # doesn't break on harmless JSON formatting changes.
-          for _ in $(seq 1 75); do
-            "$BIN" status --json > status.json || true
-            if jq -e '.state == "running" and (.vmm_pid | type == "number")' status.json >/dev/null; then
-              break
-            fi
-            sleep 0.2
-          done
-          cat status.json
-          "$BIN" status   # exit 0 == Running
-          jq -e '.state == "running" and (.vmm_pid | type == "number")' status.json
-          echo "::endgroup::"
-
-          echo "::group::stop"
-          "$BIN" stop
-          echo "::endgroup::"
-
-          echo "::group::status (expect stopped)"
-          # `status`/`status --json` exit 1 when stopped (documented), so capture
-          # without tripping set -e / pipefail, then assert structurally.
-          "$BIN" status --json > status-after.json || true
-          cat status-after.json
-          jq -e '.state == "stopped"' status-after.json
-          # Also confirm the documented non-zero exit for the stopped state.
-          if "$BIN" status; then echo "::error::expected non-zero status exit after stop"; exit 1; fi
-          echo "::endgroup::"
-
-          echo "daemon lifecycle OK"
+        run: ./scripts/lifecycle-e2e.sh
+
+      - name: Session e2e via the CLI (activate/exec/destroy; autospawns minvmd)
+        # The SAME unified proof every target lane runs (scripts/session-e2e.sh)
+        # against Deployment Model 3 (native Linux + VM — the target this
+        # lane owns; see docs/specs/03-spec-networking): from a clean state
+        # `minimal --minvmd activate` must auto-spawn minvmd, boot the microVM,
+        # and round-trip a session exec over the vsock bridge — the Linux/KVM
+        # counterpart of the macOS lane's CLI gate. E2E_PROJECT_DIR=/tmp: no
+        # project sync into the guest yet, so activate a path that exists in
+        # the guest image. minimal/minvmd resolve from the testbed on PATH.
+        env:
+          MINVMD_BOOT_LOG: ${{ runner.temp }}/cli-e2e-boot.log
+        run: E2E_VM=1 E2E_MINIMAL_ARGS=--minvmd E2E_PROJECT_DIR=/tmp ./scripts/session-e2e.sh
 
       - name: Upload guest boot console logs
         if: always()
@@ -351,10 +349,11 @@ jobs:
           path: |
             ${{ runner.temp }}/boot.log
             ${{ runner.temp }}/boot-daemon.log
+            ${{ runner.temp }}/cli-e2e-boot.log
           if-no-files-found: ignore
 
   # Single required status check for this lane (branch-protection context:
-  # ci-linux-kvm-success, joins the ruleset after the soak — #687). Green when
+  # ci-linux-kvm-success, joins the ruleset after the soak). Green when
   # build + tests succeeded OR were path-skipped; red only on failure/cancellation.
   ci-linux-kvm-success:
     if: always()

@@ -59,6 +59,8 @@ jobs:
               - "Cargo.lock"
               # The toolchain pin: a rustc bump must re-run the Rust lanes.
               - "rust-toolchain.toml"
+              # Nextest profiles change how this lane's tests run.
+              - ".config/nextest.toml"
               - "scripts/**"
               - ".minimal/**"
               - ".github/actions/setup-rust/**"