8000 [WIP] fix(minimal2): serialize concurrent attach establishment (#588) by norrietaylor · Pull Request #590 · gominimal/minimal · GitHub
[go: up one dir, main page]

Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/minimal/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ clap_complete.workspace = true
camino.workspace = true
chrono.workspace = true
dirs.workspace = true
libc.workspace = true
minimald-rpc.workspace = true
paths.workspace = true
russh.workspace = true
Expand Down
106 changes: 106 additions & 0 deletions crates/minimal/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ enum Command {
/// Proxy stdio to a daemon UDS socket (used as an SSH ProxyCommand).
#[command(hide = true)]
Proxy(ProxyArgs),
/// Hold the attach-establishment lock briefly, then exit. Internal: spawned
/// by `attach` to serialize concurrent session handshakes around the libkrun
/// vsock concurrency bug (#588). Prints `ok` once the lock is held.
#[command(hide = true)]
AttachLock(AttachLockArgs),
/// Forward a local TCP port to a remote address inside a PTask via SSH
/// (R4.8, R4.9).
///
Expand Down Expand Up @@ -263,6 +268,18 @@ struct ProxyArgs {
socket: String,
}

/// Arguments for the internal `attach-lock` helper.
#[derive(Debug, Args)]
struct AttachLockArgs {
/// Path to the establishment lock file to hold.
#[arg(long)]
lock_path: PathBuf,
/// Milliseconds to hold the lock after acquiring it (the establishment
/// window during which a concurrent attach must not race the handshake).
#[arg(long)]
window_ms: u64,
}

/// Arguments for `minimal ssh-forward`.
#[derive(Debug, Args)]
struct SshForwardArgs {
Expand Down Expand Up @@ -321,6 +338,7 @@ async fn main() -> Result<(), ()> {
MeshCommand::Leave => cmd_mesh_leave(&cli.global_args),
},
Command::Proxy(args) => cmd_proxy(args).await,
Command::AttachLock(args) => cmd_attach_lock(args),
Command::SshForward(args) => cmd_ssh_forward(&cli.global_args, args).await,
Command::Login(args) => cmd_login(&cli.global_args, args).await,
Command::Completions(CompletionsArgs { shell }) => {
Expand Down Expand Up @@ -576,6 +594,47 @@ async fn cmd_attach(global: &GlobalArgs, args: AttachArgs) -> Result<(), ()> {
// depend on socat or nc being installed.
let exe =
std::env::current_exe().map_err(|e| eprintln!("cannot determine current exe: {e}"))?;

// Serialize attach establishment to dodge the libkrun vsock concurrency bug
// (#588): concurrent session handshakes racing through the host→guest muxer
// can leave one wedged. A short-lived detached holder takes an exclusive
// `flock` for the establishment window; we wait until it reports the lock is
// held, then exec ssh. A concurrent attach blocks on the same lock until this
// window elapses, so handshakes don't overlap. Best-effort: if the helper
// can't spawn we proceed unserialized rather than block the attach.
let window_ms: u64 = std::env::var("MINIMAL_ATTACH_ESTABLISH_WINDOW_MS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(4000);
if window_ms > 0 {
let lock_path = sock
.parent()
.map(|p| p.join(".attach-establish.lock"))
.unwrap_or_else(|| std::path::PathBuf::from("/tmp/.minimal-attach-establish.lock"));
match std::process::Command::new(&exe)
.arg("attach-lock")
.arg("--lock-path")
.arg(&lock_path)
.arg("--window-ms")
.arg(window_ms.to_string())
.stdout(std::process::Stdio::piped())
.spawn()
{
Ok(mut child) => {
// Block until the holder reports `ok` (lock acquired) or dies.
if let Some(mut out) = child.stdout.take() {
use std::io::Read;
let mut buf = [0u8; 3];
let _ = out.read(&mut buf);
}
// Leave the holder running detached; it releases after the window.
}
Err(e) => {
tracing::warn!("attach-lock helper failed to spawn ({e}); attaching unserialized");
}
}
Comment on lines +623 to +635
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | 🏗️ Heavy lift

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Map relevant file structure first.
ast-grep outline crates/minimal/src/main.rs --view expanded > /tmp/main_outline.txt || true

# Show the relevant line ranges around the attach-lock flow.
sed -n '580,740p' crates/minimal/src/main.rs | cat -n

echo
echo '--- outline ---'
sed -n '1,240p' /tmp/main_outline.txt | cat -n

Repository: gominimal/minimal

Length of output: 11405


🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Locate the attach-lock helper / flock logic and any timeout or fallback handling.
rg -n -C 4 "attach-lock|flock|LOCK_EX|proceed unserialized|spawn\\(|stdout\\.take\\(|read\\(&mut buf\\)|cmd_attach_lock|unserialized" crates/minimal/src/main.rs

echo
echo '--- any wait/reap handling around detached child ---'
rg -n -C 4 "wait\\(|try_wait\\(|kill\\(|detach|zombie|reap" crates/minimal/src/main.rs

Repository: gominimal/minimal

Length of output: 5911


Bound the attach-lock wait and reap the helper

  • out.read(&mut buf) can block indefinitely here because cmd_attach_lock uses a blocking flock(LOCK_EX) with no timeout; a stale holder on the lock file will stall attach instead of degrading to the unserialized path.
  • The spawned helper is never waited on before ssh.exec(), so if it exits during the session it can sit as a zombie until the parent process exits.
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@crates/minimal/src/main.rs` around lines 623 - 635, The attach-lock flow in
the helper spawn block can hang forever on `out.read(&mut buf)` and leaves the
spawned child unreaped. Update the attach path around the
`cmd_attach_lock`/`child.stdout.take()` logic to use a bounded wait or timeout
and fall back to the unserialized path if the helper does not report readiness
promptly. Also keep a handle to the spawned child and explicitly reap it after
`ssh.exec()` (or when attach ends) so it cannot become a zombie.

}

let proxy_cmd = format!(
"{} proxy --socket {}",
shell_quote(&exe.display().to_string()),
Expand Down Expand Up @@ -617,6 +676,53 @@ async fn cmd_attach(global: &GlobalArgs, args: AttachArgs) -> Result<(), ()> {
Err(())
}

/// Internal helper spawned by `cmd_attach`: hold an exclusive establishment lock
/// for a bounded window, then exit (releasing it).
///
/// Works around the libkrun vsock concurrency bug (#588) where concurrent session
/// handshakes racing through the host→guest muxer can leave one wedged. By holding
/// this lock for the establishment window, concurrent attaches serialize their
/// handshakes (the next blocks until the current window elapses) instead of
/// racing. Prints `ok` to stdout once the lock is held so the parent can exec
/// ssh, then keeps holding it in the background until the window expires.
///
/// Uses a blocking `flock(LOCK_EX)` — the same primitive `lcache` uses for its
/// read-tracker, minus `LOCK_NB` so a contender waits rather than failing. flock
/// works on Darwin (BSD) and Linux alike, and the lock is tied to the open file
/// description, so it releases when this process exits.
fn cmd_attach_lock(args: AttachLockArgs) -> Result<(), ()> {
use std::io::Write;
use std::os::unix::io::AsRawFd;

let file = std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(false) // the file is only a flock target; never written
.open(&args.lock_path)
.map_err(|e| eprintln!("attach-lock: open {}: {e}", args.lock_path.display()))?;

// Block until the exclusive lock is held; a concurrent holder releases it
// when its own window elapses (and on process exit).
// SAFETY: `file` owns a valid open fd for the duration of this call.
let ret = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) };
if ret != 0 {
eprintln!(
"attach-lock: flock {}: {}",
args.lock_path.display(),
std::io::Error::last_os_error()
);
return Err(());
}

// Signal the parent that establishment may proceed.
println!("ok");
let _ = std::io::stdout().flush();

std::thread::sleep(std::time::Duration::from_millis(args.window_ms));
// `file` drops here, closing the fd and releasing the flock.
Ok(())
}

/// Print the effective networking policy for a session as JSON.
async fn cmd_session_policy(global: &GlobalArgs, args: PolicyArgs) -> Result<(), ()> {
if let Err(e) = autospawn::ensure_daemon_running(global.minvmd, global.minimal_dir.as_deref()) {
Expand Down
Loading
0