norrietaylor · Jul 22, 2026 · Jul 22, 2026
@@ -0,0 +1,167 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Norrie Taylor <norrie@minimal.dev>
+Date: Wed, 22 Jul 2026 16:00:00 -0700
+Subject: [PATCH] vsock: coalesce recv into full descriptors instead of
+ emitting sub-skb packets
+
+libkrun's unix-vsock proxy issues one recv() per RX descriptor and emits
+whatever that single read found buffered on the host socket. Against a bulk
+sender that hands the stream over in small chunks, the muxer drains the
+socket faster than the writer fills it, so descriptors carry a fraction of
+their capacity and the stream fragments into sub-KiB packets even though the
+peer's window is wide open.
+
+That is expensive in the one currency the receiver is short of. Linux charges
+every queued packet SKB_TRUESIZE(0) against buf_alloc regardless of payload
+-- 576 bytes on arm64 -- so a stream whose mean packet falls below that
+exhausts the receiver's queue budget long before its byte budget. With a
+256 KiB window the queue caps at 455 packets, and since 6.12.92
+("vsock/virtio: reset connection on receiving queue overflow") the receiver
+answers the 456th with a connection reset (ENOBUFS) rather than a silent
+drop. This is the failure behind bulk host->guest uploads.
+
+Fix recv_to_pkt to fill the descriptor rather than emit one recv()'s worth:
+
+  - loop reading into the descriptor until it is full or the socket returns
+    EAGAIN, so a burst already buffered is coalesced into one packet;
+
+  - and, because draining alone does not help when the writer is slower than
+    we are across wakeups (every notification then finds a few hundred bytes
+    and the loop exits on EAGAIN with the descriptor barely touched), wait
+    briefly on EAGAIN when we are still below a packet worth carrying. poll()
+    returns as soon as more data lands, so a fast writer pays nothing; a slow
+    one costs at most COALESCE_ROUNDS ms; and we always deliver what we have
+    rather than holding it, so no stream stalls on this.
+
+Measured with a guest kernel instrumented on both the rejection path and
+libkrun's emit path: on the unpatched build 86% of sub-576-byte packets came
+from the fill loop exiting on EAGAIN with descriptor space and credit both
+free -- not from the credit clamp (14%, and only in the final bytes of the
+window) and never from the descriptor size (a constant 3732 B). Waiting on
+that EAGAIN is therefore the operative lever. A/B on one host with libkrun
+the only variable: 0 of 10 uploads completed without this patch, 16 of 16
+with it, and the instrumented kernel logged no rejections at all.
+---
+diff --git a/src/devices/src/virtio/vsock/unix.rs b/src/devices/src/virtio/vsock/unix.rs
+index 2f51c05..0ebc8b0 100644
+--- a/src/devices/src/virtio/vsock/unix.rs
++++ b/src/devices/src/virtio/vsock/unix.rs
+@@ -5,13 +5,14 @@ use super::{
+
+ use nix::errno::Errno;
+ use nix::fcntl::{fcntl, FcntlArg, OFlag};
++use nix::poll::{poll, PollFd, PollFlags, PollTimeout};
+ use nix::sys::socket::{
+     accept, bind, connect, listen, recv, send, shutdown, socket, AddressFamily, Backlog, MsgFlags,
+     Shutdown, SockFlag, SockType, UnixAddr,
+ };
+ use std::collections::HashMap;
+ use std::num::Wrapping;
+-use std::os::fd::{FromRawFd, OwnedFd};
++use std::os::fd::{AsFd, FromRawFd, OwnedFd};
+ use std::os::unix::io::{AsRawFd, RawFd};
+ use std::path::PathBuf;
+ use std::sync::{Arc, Mutex};
+@@ -219,25 +220,85 @@ impl UnixProxy {
+                 return RecvPkt::WaitForCredit;
+             }
+
+-            match recv(
+-                self.fd.as_raw_fd(),
+-                &mut buf[..max_len],
+-                MsgFlags::MSG_DONTWAIT,
+-            ) {
+-                Ok(cnt) => {
+-                    debug!("recv cnt={cnt}");
+-                    if cnt > 0 {
+-                        debug!("recv rx_cnt={}", self.rx_cnt);
+-                        RecvPkt::Read(cnt)
+-                    } else {
+-                        RecvPkt::Close
+-                    }
++            // Fill the descriptor rather than emitting whatever a single
++            // recv() happened to find buffered. A bulk sender hands us the
++            // stream in small chunks, and one recv() per descriptor turns
++            // those into one vsock packet each: we outrun the writer and
++            // fragment the stream into sub-KiB packets even though the peer's
++            // window is wide open. Each packet then costs the peer a whole
++            // skb of queue accounting for a fraction of a descriptor's worth
++            // of payload.
++            //
++            // Looping until the descriptor is full or the socket is drained
++            // is safe with level-triggered polling: we never leave readable
++            // data behind on purpose, so we cannot spin on a socket we
++            // declined to read. We stop on EAGAIN (drained), on a full
++            // descriptor (progress, and the next descriptor picks up the
++            // rest), or on EOF/error once we already have bytes -- those are
++            // re-reported on the next call, with the data delivered first.
++            //
++            // Draining the socket is not enough on its own: when the writer is
++            // slower than we are, every wakeup finds only a few hundred bytes
++            // buffered and the loop below exits on EAGAIN with the descriptor
++            // barely touched. The receiver charges a whole skb of queue
++            // accounting for each of those packets -- SKB_TRUESIZE(0), 576
++            // bytes on arm64 -- so a stream of sub-576-byte packets exhausts
++            // its queue budget long before its byte budget, and since 6.12.92
++            // Linux answers that with a connection reset rather than a drop.
++            //
++            // So when a drained socket leaves us below a packet worth
++            // carrying, wait briefly for the writer to catch up instead of
++            // spending an skb on a fraction of a descriptor. poll() returns as
++            // soon as more data lands, so a fast writer pays nothing; a slow
++            // one costs at most COALESCE_ROUNDS ms, and we always deliver what
++            // we have rather than holding it indefinitely.
++            const MIN_PKT: usize = 4096;
++            const COALESCE_ROUNDS: u32 = 4;
++
++            let mut total = 0;
++            let mut waits = 0;
++            let ret = loop {
++                if total == max_len {
++                    break RecvPkt::Read(total);
+                 }
+-                Err(e) => {
+-                    debug!("recv_pkt: recv error: {e:?}");
+-                    RecvPkt::Error
++                match recv(
++                    self.fd.as_raw_fd(),
++                    &mut buf[total..max_len],
++                    MsgFlags::MSG_DONTWAIT,
++                ) {
++                    Ok(0) => {
++                        break if total == 0 {
++                            RecvPkt::Close
++                        } else {
++                            RecvPkt::Read(total)
++                        }
++                    }
++                    Ok(cnt) => total += cnt,
++                    Err(Errno::EINTR) => continue,
++                    Err(Errno::EAGAIN) => {
++                        if total == 0 {
++                            // Spurious wakeup: nothing readable after all.
++                            break RecvPkt::Error;
++                        }
++                        if total >= std::cmp::min(max_len, MIN_PKT) || waits == COALESCE_ROUNDS {
++                            break RecvPkt::Read(total);
++                        }
++                        waits += 1;
++                        let mut fds = [PollFd::new(self.fd.as_fd(), PollFlags::POLLIN)];
++                        let _ = poll(&mut fds, PollTimeout::from(1u8));
++                    }
++                    Err(e) => {
++                        debug!("recv_pkt: recv error: {e:?}");
++                        break if total == 0 {
++                            RecvPkt::Error
++                        } else {
++                            RecvPkt::Read(total)
++                        };
++                    }
+                 }
+-            }
++            };
++            debug!("recv total={total}");
++            ret
+         } else {
+             debug!("recv_pkt: pkt without buf");
+             RecvPkt::Error