twitchyliquid64 · Jul 22, 2026 · Jul 22, 2026
@@ -0,0 +1,69 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Norrie Taylor <norrie@minimal.dev>
+Date: Tue, 21 Jul 2026 20:37:36 -0700
+Subject: [PATCH] vsock: signal the used queue when requesting a credit update
+
+A proxy that runs out of peer credit mid-batch sets WaitingCreditUpdate,
+emits a VSOCK_OP_CREDIT_REQUEST, and disarms polling on its host fd so the
+connection is driven only by the peer's reply.
+
+push_packet() places that request in the used ring but does not raise the
+interrupt: the IRQ comes solely from process_proxy_update(), and only when
+ProxyUpdate::signal_queue is set. signal_queue is taken from recv_pkt()'s
+have_used, which is false exactly when no data packet was produced -- i.e.
+when the first packet of the batch is the one that has to wait.
+
+In that case the peer is never woken. It does not see the credit request,
+so it sends no credit update, and polling is already disarmed on our side,
+so nothing on the host side wakes the connection either. The stream stalls
+until an unrelated event happens to signal the queue.
+
+This is usually masked, because a batch normally pushes data packets before
+credit runs out and those set signal_queue themselves, but it is reachable
+whenever a batch begins with no credit available. Set signal_queue
+explicitly when emitting the request.
+
+tsi_stream.rs has the identical construct and the identical gap; fix both.
+---
+ src/devices/src/virtio/vsock/tsi_stream.rs |  6 ++++++
+ src/devices/src/virtio/vsock/unix.rs       | 10 ++++++++++
+ 2 files changed, 16 insertions(+)
+
+diff --git a/src/devices/src/virtio/vsock/tsi_stream.rs b/src/devices/src/virtio/vsock/tsi_stream.rs
+index c819398..b544896 100644
+--- a/src/devices/src/virtio/vsock/tsi_stream.rs
++++ b/src/devices/src/virtio/vsock/tsi_stream.rs
+@@ -834,6 +834,12 @@ impl Proxy for TsiStreamProxy {
+                         fwd_cnt: self.tx_cnt.0,
+                     };
+                     update.push_credit_req = Some(rx);
++                    // See the matching comment in unix.rs: push_packet() does
++                    // not raise the IRQ, and polling is disarmed just below,
++                    // so the request needs an explicit signal or the
++                    // connection can stall waiting for a credit update the
++                    // peer was never woken to send.
++                    update.signal_queue = true;
+                 }
+
+                 if self.status == ProxyStatus::Closed {
+diff --git a/src/devices/src/virtio/vsock/unix.rs b/src/devices/src/virtio/vsock/unix.rs
+index 2f51c05..3e622ab 100644
+--- a/src/devices/src/virtio/vsock/unix.rs
++++ b/src/devices/src/virtio/vsock/unix.rs
+@@ -576,6 +576,16 @@ impl Proxy for UnixProxy {
+                         fwd_cnt: self.tx_cnt.0,
+                     };
+                     update.push_credit_req = Some(rx);
++                    // push_packet() only places the request in the used ring;
++                    // the IRQ is raised solely by process_proxy_update() when
++                    // signal_queue is set. We disarm polling on this fd just
++                    // below, so a request delivered without an interrupt
++                    // leaves the connection stalled until some unrelated
++                    // event happens to signal the queue. Usually masked
++                    // because recv_pkt() pushed data packets first and set
++                    // signal_queue itself, but not when the very first packet
++                    // of the batch has to wait.
++                    update.signal_queue = true;
+                 }
+
+                 if self.status == ProxyStatus::Closed {
@@ -0,0 +1,119 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Norrie Taylor <norrie@minimal.dev>
+Date: Tue, 21 Jul 2026 20:43:53 -0700
+Subject: [PATCH] vsock: fill the RX descriptor instead of one recv() per
+ packet
+
+recv_to_pkt() issues a single recv() per RX descriptor and emits whatever
+that call happened to find buffered on the host socket. Against a bulk
+sender that hands the stream over in small chunks, the muxer drains the
+socket faster than the writer fills it, so each descriptor carries a
+fraction of its capacity and the stream is fragmented into sub-KiB packets.
+
+Measured on a 12.8 MB host-to-guest transfer over a unix-backed port:
+40,749 packets, mean payload 1895 B, 35.6% of reads under 1 KiB. FIONREAD
+sampled immediately before each recv() averaged 2199 bytes while the peer's
+available credit averaged 239,848 of a 262,144-byte window, and the
+zero-credit path was reached 10 times in 40,749 reads. 74% of reads drained
+the socket completely with descriptor space still free. The peer's window
+was never the constraint -- socket occupancy at the instant of the read was.
+
+Fragmentation is expensive for the peer: a Linux receiver charges every
+queued packet SKB_TRUESIZE(0) (576 bytes on arm64) against buf_alloc
+regardless of payload, so sub-KiB packets burn queue budget far faster than
+they deliver bytes.
+
+Loop until the descriptor is full or the socket returns EAGAIN. This is
+safe under level-triggered polling because we never deliberately leave
+readable data behind: every exit either drained the socket or filled a
+descriptor, so the next notification always corresponds to new data or to a
+descriptor we can immediately make progress on. EOF and errors encountered
+after some bytes were read deliver the data first and are re-reported on
+the following call.
+---
+ src/devices/src/virtio/vsock/unix.rs | 70 +++++++++++++++++++++-------
+ 1 file changed, 53 insertions(+), 17 deletions(-)
+
+diff --git a/src/devices/src/virtio/vsock/unix.rs b/src/devices/src/virtio/vsock/unix.rs
+index 3e622ab..afb65d9 100644
+--- a/src/devices/src/virtio/vsock/unix.rs
++++ b/src/devices/src/virtio/vsock/unix.rs
+@@ -219,25 +219,61 @@ impl UnixProxy {
+                 return RecvPkt::WaitForCredit;
+             }
+
+-            match recv(
+-                self.fd.as_raw_fd(),
+-                &mut buf[..max_len],
+-                MsgFlags::MSG_DONTWAIT,
+-            ) {
+-                Ok(cnt) => {
+-                    debug!("recv cnt={cnt}");
+-                    if cnt > 0 {
+-                        debug!("recv rx_cnt={}", self.rx_cnt);
+-                        RecvPkt::Read(cnt)
+-                    } else {
+-                        RecvPkt::Close
+-                    }
++            // Fill the descriptor rather than emitting whatever a single
++            // recv() happened to find buffered. A bulk sender hands us the
++            // stream in small chunks, and one recv() per descriptor turns
++            // those into one vsock packet each: we outrun the writer and
++            // fragment the stream into sub-KiB packets even though the peer's
++            // window is wide open. Each packet then costs the peer a whole
++            // skb of queue accounting for a fraction of a descriptor's worth
++            // of payload.
++            //
++            // Looping until the descriptor is full or the socket is drained
++            // is safe with level-triggered polling: we never leave readable
++            // data behind on purpose, so we cannot spin on a socket we
++            // declined to read. We stop on EAGAIN (drained), on a full
++            // descriptor (progress, and the next descriptor picks up the
++            // rest), or on EOF/error once we already have bytes -- those are
++            // re-reported on the next call, with the data delivered first.
++            let mut total = 0;
++            let ret = loop {
++                if total == max_len {
++                    break RecvPkt::Read(total);
+                 }
+-                Err(e) => {
+-                    debug!("recv_pkt: recv error: {e:?}");
+-                    RecvPkt::Error
++                match recv(
++                    self.fd.as_raw_fd(),
++                    &mut buf[total..max_len],
++                    MsgFlags::MSG_DONTWAIT,
++                ) {
++                    Ok(0) => {
++                        break if total == 0 {
++                            RecvPkt::Close
++                        } else {
++                            RecvPkt::Read(total)
++                        }
++                    }
++                    Ok(cnt) => total += cnt,
++                    Err(Errno::EINTR) => continue,
++                    Err(Errno::EAGAIN) => {
++                        break if total == 0 {
++                            // Spurious wakeup: nothing readable after all.
++                            RecvPkt::Error
++                        } else {
++                            RecvPkt::Read(total)
++                        }
++                    }
++                    Err(e) => {
++                        debug!("recv_pkt: recv error: {e:?}");
++                        break if total == 0 {
++                            RecvPkt::Error
++                        } else {
++                            RecvPkt::Read(total)
++                        };
++                    }
+                 }
+-            }
++            };
++            debug!("recv total={total}");
++            ret
+         } else {
+             debug!("recv_pkt: pkt without buf");
+             RecvPkt::Error
@@ -4,6 +4,7 @@ let gcc = import "../gcc/build.ncl" in
 let glibc = import "../glibc/build.ncl" in
 let libkrunfw = import "../libkrunfw/build.ncl" in
 let make = import "../make/build.ncl" in
+let patch = import "../patch/build.ncl" in
 let pkgconf = import "../pkgconf/build.ncl" in
 let rust = import "../rust/build.ncl" in
 let toolchain = import "../toolchain/build.ncl" in
@@ -14,6 +15,36 @@ let version = "1.19.4" in
 
   build_deps = [
     { file = "build.sh" } | Local,
+    # Two vsock fixes for a bulk host->guest stream that dies mid-upload with
+    # ENOBUFS. Order matters: 0002 is written against the tree 0001 produces,
+    # and build.sh applies them by name in that order.
+    #
+    # 0001 -- the credit-request path sets `push_credit_req` but never
+    # `signal_queue`, and push_packet() alone does not raise the IRQ. A proxy
+    # that must wait for credit therefore disarms its own polling and never
+    # wakes the peer, so the credit update it is waiting on is never sent.
+    # Usually masked (a batch normally pushes data packets first, and those
+    # set signal_queue), reachable when the first packet of a batch has to
+    # wait. A real bug, but NOT the upload fix -- measured 0/6 on its own.
+    #
+    # 0002 -- the actual fix. recv_to_pkt() issued one recv() per RX
+    # descriptor and shipped whatever happened to be buffered, so the muxer
+    # outran the writer and fragmented the stream into sub-KiB packets. A
+    # Linux peer charges every queued skb SKB_TRUESIZE(0) (576 bytes on
+    # arm64) against buf_alloc regardless of payload, so those packets burn
+    # queue budget far faster than they deliver bytes and the peer resets the
+    # connection with credit still outstanding. Instrumentation over 40,749
+    # packets: 74% of reads drained the socket with descriptor space still
+    # free, mean 2,199 bytes available against 239,848 bytes of credit -- the
+    # window was never the constraint. Loop until the descriptor is full or
+    # the socket returns EAGAIN.
+    #
+    # Byte-identical to the patches the macOS source build carries at
+    # vendor/libkrun/patches/ in gominimal/minimal; drop both once they land
+    # in an upstream release we can pin.
+    # https://github.com/gominimal/minimal/issues/869
+    { file = "0001-vsock-signal-the-used-queue-when-requesting-credit.patch" } | Local,
+    { file = "0002-vsock-fill-the-rx-descriptor-instead-of-one-recv-per-packet.patch" } | Local,
     {
       url = "https://github.com/containers/libkrun/archive/refs/tags/v%{version}.tar.gz",
       sha256 = "e8775fab2b460972a67ca6cd936296bb79cdb078d852d712a283cb290dd0b284",
@@ -23,6 +54,7 @@ let version = "1.19.4" in
     base,
     toolchain,
     make,
+    patch,
     rust,
     pkgconf,
   ],

@@ -7,6 +7,15 @@ export CC=gcc
 export LD=gcc
 export RUSTFLAGS="-C linker=gcc --remap-path-prefix=$(pwd)=/builddir --remap-path-prefix=$HOME/.cargo=/cargo"
 
+# vsock fixes (see build.ncl for what each one does). Applied by name in a
+# fixed order rather than by glob: 0002 is written against the tree 0001
+# produces, so the sequence is load-bearing and should not depend on how the
+# shell happens to sort a wildcard. `set -e` plus patch's non-zero exit on a
+# rejected hunk makes a stale patch abort the build; a silently-skipped patch
+# would publish a libkrun that looks fixed and is not.
+patch -Np1 -i "0001-vsock-signal-the-used-queue-when-requesting-credit.patch"
+patch -Np1 -i "0002-vsock-fill-the-rx-descriptor-instead-of-one-recv-per-packet.patch"
+
 # BLK=1 enables virtio-blk (exports krun_add_disk2); `blk` is not a default
 # libkrun feature, so consumers fail to link without it.
 make CC=gcc BLK=1 -j"$(nproc)"