From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 4CB441FF0DF for ; Thu, 05 Nov 2026 15:50:45 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 0089B215AD; Thu, 05 Nov 2026 15:50:42 +0100 (CET) From: Filip Schauer To: pve-devel@lists.proxmox.com Subject: [RFC pve-lxc-console v2 1/2] initial commit Date: Thu, 5 Nov 2026 15:49:43 +0100 Message-ID: <20261105144947.124148-2-f.schauer@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261105144947.124148-1-f.schauer@proxmox.com> References: <20261105144947.124148-1-f.schauer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1793890222200 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.291 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_SHORT 0.001 Use of a URL Shortener for very short URL RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: VMDES5P5FF45JNDCLQHWGHZJRKQ45S75 X-Message-ID-Hash: VMDES5P5FF45JNDCLQHWGHZJRKQ45S75 X-MailFrom: f.schauer@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: pve-lxc-console is a single binary that replaces the combination of dtach and lxc-console used to provide container consoles in PVE. The master owns the container console and serves any number of clients over the unix socket /var/run/dtach/vzctlconsole, speaking the dtach protocol. The master keeps a 32 KiB scrollback buffer and replays it to every client that attaches. The console is accessed through liblxc directly instead of spawning lxc-console. The exact behaviour depends on the console mode (cmode): * console: If LXC has a console buffer configured, the console fd is only used as a doorbell and the output is read from LXC's ring buffer. This also covers everything printed since container start, so the history survives a late start of the master. If no console buffer is available, the fd is read as the data stream, like lxc-console did. * tty: the console fd of a free tty is the data stream. Output printed before the master starts is buffered internally by the kernel until it is read. * shell: a pty running `lxc-attach`. The shell is only started when the console is opened for the first time, so there is no earlier output to preserve. Signed-off-by: Filip Schauer --- The development of this tool was assisted by LLMs. .cargo/config.toml | 5 + .gitignore | 12 ++ Cargo.toml | 14 ++ Makefile | 77 +++++++++ build.rs | 49 ++++++ debian/changelog | 5 + debian/control | 31 ++++ debian/copyright | 16 ++ debian/rules | 29 ++++ debian/source/format | 1 + src/client.rs | 97 +++++++++++ src/lxc.rs | 85 +++++++++ src/main.rs | 91 ++++++++++ src/master.rs | 402 +++++++++++++++++++++++++++++++++++++++++++ src/packet.rs | 85 +++++++++ 15 files changed, 999 insertions(+) create mode 100644 .cargo/config.toml create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 Makefile create mode 100644 build.rs create mode 100644 debian/changelog create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/rules create mode 100644 debian/source/format create mode 100644 src/client.rs create mode 100644 src/lxc.rs create mode 100644 src/main.rs create mode 100644 src/master.rs create mode 100644 src/packet.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..3b5b6e4 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +[source] +[source.debian-packages] +directory = "/usr/share/cargo/registry" +[source.crates-io] +replace-with = "debian-packages" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d42ee21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +Cargo.lock +/target +**/*.rs.bk +*.buildinfo +*.changes +*.deb +*.dsc +*.tar.?z +*.tar.zst +/*-deb +/*-dsc +/pve-lxc-console-[0-9]*/ diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..563ba62 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "pve-lxc-console" +version = "1.0.0" +edition = "2021" +description = "Console master/client for LXC containers" +license = "AGPL-3" + +[dependencies] +anyhow = "1" +nix = { version = "0.29", features = ["event", "fs", "poll", "process", "signal", "term"] } +pico-args = "0.5" + +[build-dependencies] +bindgen = { version = "0.71", default-features = false, features = ["runtime"] } diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6713951 --- /dev/null +++ b/Makefile @@ -0,0 +1,77 @@ +include /usr/share/dpkg/architecture.mk +include /usr/share/dpkg/pkg-info.mk + +PACKAGE=pve-lxc-console +ARCH := $(DEB_BUILD_ARCH) + +OUTPUT_DIR := +BUILD_DIR := $(OUTPUT_DIR)$(PACKAGE)-$(DEB_VERSION) + +DEB = $(PACKAGE)_$(DEB_VERSION)_$(ARCH).deb +DSC = $(PACKAGE)_$(DEB_VERSION).dsc + +TARGET_DIR := target/debug + +ifeq ($(BUILD_MODE), release) +CARGO_BUILD_ARGS += --release +TARGET_DIR := target/release +endif + +CARGO ?= cargo + +.PHONY: all build +all: build + +build: $(TARGET_DIR)/pve-lxc-console +$(TARGET_DIR)/pve-lxc-console: build.rs Cargo.toml src/ + $(CARGO) build $(CARGO_BUILD_ARGS) + +.PHONY: install +install: $(TARGET_DIR)/pve-lxc-console + install -dm755 $(DESTDIR)/usr/bin + install -m755 $(TARGET_DIR)/pve-lxc-console $(DESTDIR)/usr/bin/ + +$(BUILD_DIR): + rm -rf $@ $@.tmp + mkdir -p $@.tmp + cp -a debian src build.rs Cargo.toml $@.tmp/ + mv $@.tmp $@ + +.PHONY: build-dir +build-dir: + rm -rf $(BUILD_DIR) + $(MAKE) $(BUILD_DIR) + +$(OUTPUT_DIR)$(DEB): $(BUILD_DIR) + cd $(BUILD_DIR); dpkg-buildpackage -b -uc -us + lintian $(OUTPUT_DIR)$(DEB) + +.PHONY: deb +deb: + rm -rf $(BUILD_DIR) + $(MAKE) $(OUTPUT_DIR)$(DEB) + +.PHONY: dsc +dsc: + rm -rf $(BUILD_DIR) + $(MAKE) $(OUTPUT_DIR)$(DSC) + lintian $(OUTPUT_DIR)$(DSC) + +$(OUTPUT_DIR)$(DSC): $(BUILD_DIR) + cd $(BUILD_DIR) && dpkg-buildpackage -S -uc -us -d + +sbuild: $(OUTPUT_DIR)$(DSC) + [ -z "$(OUTPUT_DIR)" ] || cd $(OUTPUT_DIR); sbuild $(DSC) + +.PHONY: clean +clean: + rm -rf $(BUILD_DIR) $(PACKAGE)-[0-9]*/ + [ -z "$(OUTPUT_DIR)" ] || rm -rf $(OUTPUT_DIR) + rm -f *.deb *.dsc *.buildinfo *.build *.changes $(PACKAGE)*.tar* + $(CARGO) clean + +.PHONY: upload +upload: UPLOAD_DIST ?= $(DEB_DISTRIBUTION) +upload: $(OUTPUT_DIR)$(DEB) + [ -z "$(OUTPUT_DIR)" ] || cd $(OUTPUT_DIR); \ + tar cf - $(DEB) | ssh -X repoman@repo.proxmox.com upload --product pve --dist $(UPLOAD_DIST) diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..773ede0 --- /dev/null +++ b/build.rs @@ -0,0 +1,49 @@ +//! Links liblxc and generates the FFI bindings from its public header. + +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn include_args() -> Vec { + println!("cargo:rerun-if-env-changed=LXC_INCLUDE_DIR"); + match env::var("LXC_INCLUDE_DIR") { + Ok(dir) if !dir.is_empty() => return vec![format!("-I{dir}")], + _ => {} + } + match Command::new("pkg-config") + .args(["--cflags-only-I", "lxc"]) + .output() + { + Ok(out) if out.status.success() => String::from_utf8_lossy(&out.stdout) + .split_whitespace() + .map(str::to_string) + .collect(), + _ => Vec::new(), + } +} + +fn main() { + println!("cargo:rerun-if-env-changed=LXC_LIB_PATH"); + match env::var("LXC_LIB_PATH") { + Ok(p) if !p.is_empty() => println!("cargo:rustc-link-arg={p}"), + _ => println!("cargo:rustc-link-lib=lxc"), + } + + let bindings = bindgen::Builder::default() + .header_contents("wrapper.h", "#include \n") + .clang_args(include_args()) + .allowlist_type("lxc_container") + .allowlist_type("lxc_console_log") + .allowlist_function("lxc_container_new") + .allowlist_function("lxc_container_put") + .rust_edition(bindgen::RustEdition::Edition2021) + .formatter(bindgen::Formatter::None) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .expect("bindgen failed: is installed (liblxc-dev)? Set LXC_INCLUDE_DIR if it is in a non-standard place"); + + let out = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")).join("lxc_bindings.rs"); + bindings + .write_to_file(out) + .expect("cannot write the bindings"); +} diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..e893854 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +pve-lxc-console (1.0.0) pve; urgency=medium + + * initial release + + -- Proxmox Support Team Tue, 27 Oct 2026 12:00:00 +0100 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..affe905 --- /dev/null +++ b/debian/control @@ -0,0 +1,31 @@ +Source: pve-lxc-console +Section: admin +Priority: optional +Build-Depends: debhelper-compat (= 13), + dh-cargo, + libclang-dev, + librust-anyhow-1+default-dev, + librust-bindgen-dev (>= 0.71.1), + librust-nix-dev (>= 0.29.0), + librust-pico-args-0.5+default-dev, + lxc-pve-dev, + pkg-config, +Maintainer: Proxmox Support Team +Standards-Version: 4.6.2 +Vcs-Git: git://git.proxmox.com/git/pve-lxc-console.git +Vcs-Browser: https://git.proxmox.com/?p=pve-lxc-console.git;a=summary +Homepage: https://www.proxmox.com +Rules-Requires-Root: no + +Package: pve-lxc-console +Architecture: any +Built-Using: ${cargo:Built-Using} +Static-Built-Using: ${cargo:Static-Built-Using} +Depends: lxc-pve, + ${misc:Depends}, + ${shlibs:Depends}, +Description: Console master/client for LXC containers + One binary that replaces dtach and lxc-console for PVE container consoles. + It attaches the terminal to the console master for a container, starting the + master first if there is none. The master owns the console, keeps a scrollback + and serves any number of clients over a dtach-compatible unix socket. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..1777df7 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,16 @@ +Copyright (C) 2026 Proxmox Server Solutions GmbH + +This software is written by Proxmox Server Solutions GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . diff --git a/debian/rules b/debian/rules new file mode 100644 index 0000000..a3c200f --- /dev/null +++ b/debian/rules @@ -0,0 +1,29 @@ +#!/usr/bin/make -f +# See debhelper(7) (uncomment to enable) +# output every command that modifies files on the build system. +DH_VERBOSE = 1 + +include /usr/share/dpkg/pkg-info.mk +include /usr/share/rustc/architecture.mk + +export BUILD_MODE=release + +CARGO=/usr/share/cargo/bin/cargo + +export CFLAGS CXXFLAGS CPPFLAGS LDFLAGS +export DEB_HOST_RUST_TYPE DEB_HOST_GNU_TYPE +export CARGO_HOME = $(CURDIR)/debian/cargo_home + +export DEB_CARGO_CRATE=pve-lxc-console_$(DEB_VERSION_UPSTREAM) +export DEB_CARGO_PACKAGE=pve-lxc-console + +%: + dh $@ --buildsystem=cargo + +override_dh_auto_configure: + @perl -ne 'if (/^version\s*=\s*"(\d+(?:\.\d+)+)"/) { my $$v_cargo = $$1; my $$v_deb = "$(DEB_VERSION_UPSTREAM)"; \ + die "ERROR: d/changelog <-> Cargo.toml version mismatch: $$v_cargo != $$v_deb\n" if $$v_cargo ne $$v_deb; exit(0); }' Cargo.toml + $(CARGO) prepare-debian $(CURDIR)/debian/cargo_registry --link-from-system + +override_dh_auto_test: + $(CARGO) test --release diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/src/client.rs b/src/client.rs new file mode 100644 index 0000000..f9fbab0 --- /dev/null +++ b/src/client.rs @@ -0,0 +1,97 @@ +use std::io::{stdin, stdout, Read, Write}; +use std::os::fd::{AsFd, AsRawFd}; +use std::os::unix::net::UnixStream; + +use anyhow::{bail, Context}; +use nix::errno::Errno; +use nix::libc; +use nix::poll::{poll, PollFd, PollFlags, PollTimeout}; +use nix::sys::signal::{SigSet, Signal}; +use nix::sys::signalfd::{SfdFlags, SignalFd}; +use nix::sys::termios::{cfmakeraw, tcgetattr, tcsetattr, SetArg}; +use nix::unistd::read; + +use crate::packet::{Packet, RedrawMethod}; + +fn get_winsize() -> (u16, u16) { + let mut winsize = libc::winsize { + ws_row: 0, + ws_col: 0, + ws_xpixel: 0, + ws_ypixel: 0, + }; + unsafe { libc::ioctl(libc::STDIN_FILENO, libc::TIOCGWINSZ, &mut winsize) }; + (winsize.ws_row, winsize.ws_col) +} + +pub fn run(mut stream: &UnixStream) -> anyhow::Result<()> { + let stdin = stdin(); + let mut stdout = stdout(); + + // Switch the terminal to raw mode + let mut attr = tcgetattr(&stdin)?; + cfmakeraw(&mut attr); + tcsetattr(&stdin, SetArg::TCSAFLUSH, &attr)?; + + let mut sigset = SigSet::empty(); + sigset.add(Signal::SIGWINCH); + sigset.thread_block()?; + let sig = SignalFd::with_flags(&sigset, SfdFlags::SFD_NONBLOCK | SfdFlags::SFD_CLOEXEC)?; + + let (rows, cols) = get_winsize(); + stream.write_all(&Packet::Attach.serialize())?; + stream.write_all( + &Packet::Redraw { + method: RedrawMethod::Winch, + rows, + cols, + } + .serialize(), + )?; + + let mut buf = [0u8; 65536]; + let mut pollfds = [ + PollFd::new(stdin.as_fd(), PollFlags::POLLIN), + PollFd::new(stream.as_fd(), PollFlags::POLLIN), + PollFd::new(sig.as_fd(), PollFlags::POLLIN), + ]; + loop { + match poll(&mut pollfds, PollTimeout::NONE) { + Ok(_) | Err(Errno::EINTR) => {} + Err(e) => bail!("poll: {e}"), + } + if pollfds[1].any() == Some(true) { + // output from the master + let Ok(n @ 1..) = stream.read(&mut buf) else { + bail!("the console master terminated"); + }; + stdout.write_all(&buf[..n])?; + stdout.flush()?; + } + if pollfds[2].any() == Some(true) { + // SIGWINCH + let _ = sig.read_signal(); + let (rows, cols) = get_winsize(); + stream + .write_all(&Packet::Winch { rows, cols }.serialize()) + .context("the console master terminated")?; + } + if pollfds[0].any() == Some(true) { + // keyboard + match read(stdin.as_raw_fd(), &mut buf[..4096]) { + Ok(0) => break, + Ok(n) => { + for chunk in buf[..n].chunks(8) { + stream + .write_all(&Packet::Push(chunk).serialize()) + .context("the console master terminated")?; + } + } + Err(Errno::EINTR) => {} + Err(_) => break, + } + } + } + let _ = stream.write_all(&Packet::Detach.serialize()); + Ok(()) +} diff --git a/src/lxc.rs b/src/lxc.rs new file mode 100644 index 0000000..a2cfad4 --- /dev/null +++ b/src/lxc.rs @@ -0,0 +1,85 @@ +use std::ffi::{c_int, CString}; +use std::io; +use std::os::unix::io::{FromRawFd, OwnedFd}; + +use anyhow::{bail, Context}; +use nix::libc; + +#[allow(non_camel_case_types, non_upper_case_globals, dead_code)] +mod ffi { + include!(concat!(env!("OUT_DIR"), "/lxc_bindings.rs")); +} + +use ffi::{lxc_console_log, lxc_container, lxc_container_new, lxc_container_put}; + +pub struct Container(*mut lxc_container); + +impl Container { + pub fn open(name: &str) -> anyhow::Result { + let cname = CString::new(name).context("invalid container name")?; + let ptr = unsafe { lxc_container_new(cname.as_ptr(), std::ptr::null()) }; + if ptr.is_null() { + bail!("lxc_container_new({name}) failed"); + } + Ok(Self(ptr)) + } + + pub fn is_running(&self) -> bool { + unsafe { ((*self.0).is_running.unwrap())(self.0) } + } + + pub fn init_pid(&self) -> i32 { + unsafe { ((*self.0).init_pid.unwrap())(self.0) } + } + + pub fn console_getfd(&self, ttynum: i32) -> io::Result<(OwnedFd, OwnedFd)> { + let mut tn: c_int = ttynum; + let mut ptx: c_int = -1; + let tty = unsafe { ((*self.0).console_getfd.unwrap())(self.0, &mut tn, &mut ptx) }; + if tty < 0 { + return Err(io::Error::last_os_error()); + } + let tty = unsafe { OwnedFd::from_raw_fd(tty) }; + + if ptx < 0 { + return Err(io::Error::last_os_error()); + } + let ptx = unsafe { OwnedFd::from_raw_fd(ptx) }; + + Ok((tty, ptx)) + } + + pub fn drain_ring(&self) -> io::Result>> { + let mut read_max: u64 = 0; + let mut log = lxc_console_log { + clear: false, + read: true, + read_max: &mut read_max, + data: std::ptr::null_mut(), + }; + let ret = unsafe { ((*self.0).console_log.unwrap())(self.0, &mut log) }; + if ret == 0 { + let data = (!log.data.is_null() && read_max > 0).then(|| unsafe { + std::slice::from_raw_parts(log.data.cast::(), read_max as usize).to_vec() + }); + unsafe { libc::free(log.data.cast()) }; + return Ok(Some(data.unwrap_or_default())); + } + let err = if ret == -1 { + io::Error::last_os_error().raw_os_error().unwrap_or(0) + } else { + -ret + }; + match err { + libc::ENODATA | libc::EMSGSIZE => Ok(Some(Vec::new())), + libc::EFAULT | libc::ENOENT => Ok(None), + e => Err(io::Error::from_raw_os_error(e)), + } + } +} + +impl Drop for Container { + fn drop(&mut self) { + unsafe { lxc_container_put(self.0) }; + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..e9828a9 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,91 @@ +use nix::fcntl::{Flock, FlockArg}; +use nix::unistd::{dup2, fork, setsid, ForkResult}; +use std::fs::{self, OpenOptions}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; + +use anyhow::bail; + +mod client; +mod lxc; +mod master; +mod packet; + +use master::ConsoleMode; + +/// Make sure a master serves `socket`, forking one off otherwise, +/// and return a client connection to it. +fn connect_master(ctid: u32, cmode: ConsoleMode, socket: &str) -> anyhow::Result { + if let Ok(stream) = UnixStream::connect(socket) { + return Ok(stream); // A master is serving it already + } + let lock = OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .open(format!("{socket}.lock"))?; + let lock = Flock::lock(lock, FlockArg::LockExclusive).map_err(|(_, e)| e)?; + if let Ok(stream) = UnixStream::connect(socket) { + return Ok(stream); // Somebody else just started it + } + + // Cleanup a leftover socket from a master that died + let _ = fs::remove_file(socket); + + let listener = UnixListener::bind(socket)?; + let _ = fs::set_permissions(socket, fs::Permissions::from_mode(0o600)); + drop(lock); + + if let ForkResult::Child = unsafe { fork() }? { + // Become the master + let _ = setsid(); + if let Ok(null) = OpenOptions::new().read(true).write(true).open("/dev/null") { + let _ = dup2(null.as_raw_fd(), 0); + let _ = dup2(null.as_raw_fd(), 1); + } + let code = match master::run(ctid, cmode, listener) { + Ok(()) => 0, + Err(e) => { + eprintln!("{e:?}"); + 1 + } + }; + let _ = fs::remove_file(socket); + std::process::exit(code); + } + + Ok(UnixStream::connect(socket)?) +} + +struct CmdArgs { + cmode: ConsoleMode, + ctid: u32, +} + +fn parse_args() -> anyhow::Result { + let mut args = pico_args::Arguments::from_env(); + + if args.contains(["-h", "--help"]) { + println!("Usage: pve-lxc-console -m "); + std::process::exit(0); + } + + let cmode = args.value_from_str("-m")?; + let ctid: u32 = args.free_from_str()?; + + if !args.finish().is_empty() { + bail!("unexpected extra arguments, use '-h' for usage"); + } + + Ok(CmdArgs { cmode, ctid }) +} + +fn main() -> anyhow::Result<()> { + let args = parse_args()?; + + let socket = format!("/var/run/dtach/vzctlconsole{}", args.ctid); + + let connection = connect_master(args.ctid, args.cmode, &socket)?; + client::run(&connection) +} diff --git a/src/master.rs b/src/master.rs new file mode 100644 index 0000000..48679c6 --- /dev/null +++ b/src/master.rs @@ -0,0 +1,402 @@ +//! The master owns the container console and serves any number of dtach-protocol clients. +//! +//! Behaviour depends on the cmode (console mode): +//! +//! * `console` +//! * With LXC console buffer enabled: the fd is only a doorbell and output is pulled from +//! LXC's ring buffer (everything since container start, so history survives a late start) +//! * Without LXC console buffer enabled: the fd is the data stream itself (backwards compat) +//! * `tty`: the fd is the data stream itself +//! * `shell`: pty running `lxc-attach` + +use std::collections::{HashMap, VecDeque}; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::CommandExt; +use std::process::Command; +use std::str::FromStr; + +use anyhow::{bail, Context}; +use nix::fcntl::{fcntl, FcntlArg, OFlag}; +use nix::libc; +use nix::pty::openpty; +use nix::sys::epoll::{Epoll, EpollCreateFlags, EpollEvent, EpollFlags, EpollTimeout}; +use nix::sys::signal::{kill, killpg, Signal}; +use nix::unistd::{tcgetpgrp, Pid}; + +use crate::lxc::Container; +use crate::packet::{Packet, RedrawMethod, PACKET_SIZE}; + +/// Size of the scrollback buffer replayed to every client that attaches +const SCROLLBACK: usize = 32 * 1024; +/// Input waiting for the container console/pty before dropping excess +const INPUT_LIMIT: usize = 1024 * 1024; + +#[derive(PartialEq, Eq)] +pub enum ConsoleMode { + Console, + Tty, + Shell, +} + +impl FromStr for ConsoleMode { + type Err = anyhow::Error; + fn from_str(s: &str) -> anyhow::Result { + match s { + "console" => Ok(Self::Console), + "tty" => Ok(Self::Tty), + "shell" => Ok(Self::Shell), + _ => bail!("unknown cmode '{s}' (expected console, tty or shell)"), + } + } +} + +enum Backend { + Console { container: Container, ring: bool }, + Tty, + Shell, +} + +/// Appends `p` to `scrollback`, evicting the oldest bytes beyond the fixed capacity. +fn feed(scrollback: &mut VecDeque, p: &[u8]) { + let p = &p[p.len().saturating_sub(SCROLLBACK)..]; // only the tail can survive + let excess = (scrollback.len() + p.len()).saturating_sub(SCROLLBACK); + scrollback.drain(..excess); + scrollback.extend(p); +} + +/// Outbound buffer with non-blocking write logic and epoll EPOLLOUT toggling. +#[derive(Default)] +struct WriteQueue { + queue: VecDeque, + congested: bool, +} + +impl WriteQueue { + fn flush(&mut self, fd: &mut File, ep: &Epoll) -> io::Result<()> { + while !self.queue.is_empty() { + match fd.write(self.queue.as_slices().0) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), + Ok(n) => { + self.queue.drain(..n); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) => return Err(e), + } + } + let more = !self.queue.is_empty(); + if more != self.congested { + self.congested = more; + let mut flags = EpollFlags::EPOLLIN; + flags.set(EpollFlags::EPOLLOUT, more); + ep.modify(&*fd, &mut EpollEvent::new(flags, fd.as_raw_fd() as u64))?; + } + Ok(()) + } +} + +struct Client { + stream: UnixStream, + attached: bool, + inbuf: [u8; PACKET_SIZE], + in_len: usize, +} + +impl Client { + fn new(stream: UnixStream) -> Self { + Self { + stream, + attached: false, + inbuf: [0; PACKET_SIZE], + in_len: 0, + } + } + + fn read_packet(&mut self) -> io::Result> { + loop { + match self.stream.read(&mut self.inbuf[self.in_len..]) { + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(n) => { + self.in_len += n; + if self.in_len == PACKET_SIZE { + self.in_len = 0; + return Ok(Some(self.inbuf)); + } + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(None), + Err(e) => return Err(e), + } + } + } + + fn send(&mut self, data: &[u8]) -> bool { + self.stream.write_all(data).is_ok() + } +} + +struct Master { + console: File, + backend: Backend, + console_in: WriteQueue, + ep: Epoll, + listener: UnixListener, + clients: HashMap, + scrollback: VecDeque, + quit: bool, +} + +/// A pty running `lxc-attach` (the `shell` console mode). +fn spawn_shell(ctid: u32) -> anyhow::Result { + let ws = libc::winsize { + ws_row: 24, + ws_col: 80, + ws_xpixel: 0, + ws_ypixel: 0, + }; + let pty = openpty(Some(&ws), None)?; + let mut cmd = Command::new("lxc-attach"); + cmd.args(["--clear-env", "-n", &ctid.to_string()]) + .stdin(pty.slave.try_clone()?) + .stdout(pty.slave.try_clone()?) + .stderr(pty.slave); + unsafe { + cmd.pre_exec(|| { + libc::setsid(); + libc::ioctl(0, libc::TIOCSCTTY, 0); + Ok(()) + }); + } + // not waited for: when it exits the master exits too + #[allow(clippy::zombie_processes)] + cmd.spawn().context("cannot run lxc-attach")?; + Ok(File::from(pty.master)) +} + +pub fn run(ctid: u32, cmode: ConsoleMode, listener: UnixListener) -> anyhow::Result<()> { + let mut scrollback = VecDeque::with_capacity(SCROLLBACK); + + // The tty fd is kept for the master's lifetime. + let (console, backend, _tty) = match cmode { + ConsoleMode::Console | ConsoleMode::Tty => { + let is_console = cmode == ConsoleMode::Console; + let c = Container::open(&ctid.to_string())?; + if !c.is_running() { + bail!("container {ctid} is not running"); + } + let (tty, ptx) = c + .console_getfd(if is_console { 0 } else { -1 }) + .with_context(|| { + if is_console { + format!("cannot attach to the console of {ctid}") + } else { + format!("no free tty in {ctid}") + } + })?; + let console = File::from(ptx); // shared only with the monitor's copy: safe to nonblock + let backend = if is_console { + // Prime the scrollback with the ring's content so far. + let ring = c + .drain_ring()? + .inspect(|d| feed(&mut scrollback, d)) + .is_some(); + Backend::Console { container: c, ring } + } else { + Backend::Tty + }; + (console, backend, Some(tty)) + } + ConsoleMode::Shell => (spawn_shell(ctid)?, Backend::Shell, None), + }; + + let console_fd = console.as_raw_fd(); + let fl = OFlag::from_bits_truncate(fcntl(console_fd, FcntlArg::F_GETFL)?); + fcntl(console_fd, FcntlArg::F_SETFL(fl | OFlag::O_NONBLOCK))?; + + listener.set_nonblocking(true)?; + + // event loop plumbing + let ep = Epoll::new(EpollCreateFlags::empty())?; + ep.add( + &listener, + EpollEvent::new(EpollFlags::EPOLLIN, listener.as_raw_fd() as u64), + )?; + ep.add( + &console, + EpollEvent::new(EpollFlags::EPOLLIN, console_fd as u64), + )?; + + Master { + console, + backend, + console_in: WriteQueue::default(), + ep, + listener, + clients: HashMap::new(), + scrollback, + quit: false, + } + .event_loop() +} + +impl Master { + fn event_loop(&mut self) -> anyhow::Result<()> { + let mut events = [EpollEvent::empty(); 32]; + while !self.quit { + let n = self.ep.wait(&mut events, EpollTimeout::NONE)?; + for ev in &events[..n] { + let id = ev.data(); + if id == self.listener.as_raw_fd() as u64 { + self.on_listen(); + } else if id == self.console.as_raw_fd() as u64 { + self.on_console_readable(); + self.flush_input(); + } else { + self.on_client_readable(id); + } + } + } + Ok(()) + } + + fn on_console_readable(&mut self) { + let mut buf = [0u8; 65536]; + let ring = matches!(self.backend, Backend::Console { ring: true, .. }); + let mut rang = false; + for _ in 0..64 { + match self.console.read(&mut buf) { + Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Ok(0) | Err(_) => return self.console_gone(), + Ok(n) if !ring => self.ingest(&buf[..n]), + Ok(_) => rang = true, + } + } + if rang { + self.do_drain(); + } + } + + fn console_gone(&mut self) { + self.do_drain(); // best effort: the monitor may already be gone + self.quit = true; + } + + fn do_drain(&mut self) { + let Backend::Console { + container, + ring: true, + } = &self.backend + else { + return; + }; + match container.drain_ring() { + Ok(Some(d)) => self.ingest(&d), + Err(_) if !container.is_running() => self.quit = true, + _ => {} + } + } + + fn ingest(&mut self, data: &[u8]) { + feed(&mut self.scrollback, data); + self.clients.retain(|_, c| !c.attached || c.send(data)); + } + + fn flush_input(&mut self) { + if self.console_in.flush(&mut self.console, &self.ep).is_err() { + self.console_gone(); + } + } + + fn set_size(&mut self, rows: u16, cols: u16, redraw: bool) { + if rows == 0 || cols == 0 { + return; + } + unsafe { + libc::ioctl( + self.console.as_raw_fd(), + libc::TIOCSWINSZ, + &libc::winsize { + ws_row: rows, + ws_col: cols, + ws_xpixel: 0, + ws_ypixel: 0, + }, + ) + }; + match &self.backend { + Backend::Console { container, .. } => { + // The monitor copies the proxy pty's size to the real console only on SIGWINCH + // (a plain ioctl has no effect); it is the parent of the container's init. + let Ok(s) = fs::read_to_string(format!("/proc/{}/stat", container.init_pid())) + else { + return; + }; + let rest = &s[s.rfind(')').map_or(0, |i| i + 2)..]; // comm may contain spaces + if let Some(monitor) = rest.split_whitespace().nth(1).and_then(|p| p.parse().ok()) { + let _ = kill(Pid::from_raw(monitor), Signal::SIGWINCH); + } + } + Backend::Shell if redraw => { + // like dtach: poke the foreground process group so a full-screen app repaints + let _ = tcgetpgrp(&self.console).and_then(|pg| killpg(pg, Signal::SIGWINCH)); + } + _ => {} + } + } + + fn on_listen(&mut self) { + while let Ok((stream, _)) = self.listener.accept() { + let _ = stream.set_nonblocking(true); + let id = stream.as_raw_fd() as u64; + let ev = EpollEvent::new(EpollFlags::EPOLLIN, id); + if self.ep.add(&stream, ev).is_ok() { + self.clients.insert(id, Client::new(stream)); + } + } + } + + fn on_client_readable(&mut self, id: u64) { + while let Some(c) = self.clients.get_mut(&id) { + let raw_packet = match c.read_packet() { + Ok(Some(raw)) => raw, + Ok(None) => return, + Err(_) => { + self.clients.remove(&id); + return; + } + }; + let Ok(packet) = Packet::try_parse(&raw_packet) else { + continue; + }; + match packet { + Packet::Push(data) => { + let space = INPUT_LIMIT.saturating_sub(self.console_in.queue.len()); + self.console_in.queue.extend(data.iter().take(space)); + self.flush_input(); + } + Packet::Attach => { + self.do_drain(); // include everything printed up to now + let snap = self.scrollback.make_contiguous(); + if let Some(c) = self.clients.get_mut(&id) { + if c.send(snap) { + c.attached = true; + } else { + self.clients.remove(&id); + } + } + } + Packet::Detach => c.attached = false, + Packet::Winch { rows, cols } => self.set_size(rows, cols, false), + Packet::Redraw { method, rows, cols } => { + if method != RedrawMethod::None { + self.set_size(rows, cols, true) + } + } + } + } + } +} diff --git a/src/packet.rs b/src/packet.rs new file mode 100644 index 0000000..14c4bea --- /dev/null +++ b/src/packet.rs @@ -0,0 +1,85 @@ +pub const PACKET_SIZE: usize = 10; // type, arg, 8 data bytes + +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum RedrawMethod { + Unspec = 0, + None = 1, + CtrlL = 2, + Winch = 3, +} + +impl TryFrom for RedrawMethod { + type Error = (); + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Unspec), + 1 => Ok(Self::None), + 2 => Ok(Self::CtrlL), + 3 => Ok(Self::Winch), + _ => Err(()), + } + } +} + +pub enum Packet<'a> { + Push(&'a [u8]), + Attach, + Detach, + Winch { + rows: u16, + cols: u16, + }, + Redraw { + method: RedrawMethod, + rows: u16, + cols: u16, + }, +} + +impl<'a> Packet<'a> { + pub fn try_parse(raw: &'a [u8; PACKET_SIZE]) -> Result { + let (msg_type, arg, data) = (raw[0], raw[1], &raw[2..]); + match msg_type { + 0 if arg <= 8 => Ok(Self::Push(&data[..arg.into()])), + 1 => Ok(Self::Attach), + 2 => Ok(Self::Detach), + 3 => Ok(Self::Winch { + rows: u16::from_ne_bytes([data[0], data[1]]), + cols: u16::from_ne_bytes([data[2], data[3]]), + }), + 4 => Ok(Self::Redraw { + method: arg.try_into()?, + rows: u16::from_ne_bytes([data[0], data[1]]), + cols: u16::from_ne_bytes([data[2], data[3]]), + }), + _ => Err(()), + } + } + + pub fn serialize(&self) -> [u8; PACKET_SIZE] { + let mut raw = [0u8; PACKET_SIZE]; + match self { + Self::Push(data) => { + raw[1] = data.len() as u8; + raw[2..2 + data.len()].copy_from_slice(data); + } + Self::Attach => raw[0] = 1, + Self::Detach => raw[0] = 2, + Self::Winch { rows, cols } => { + raw[0] = 3; + raw[2..4].copy_from_slice(&rows.to_ne_bytes()); + raw[4..6].copy_from_slice(&cols.to_ne_bytes()); + } + Self::Redraw { method, rows, cols } => { + raw[0] = 4; + raw[1] = *method as u8; + raw[2..4].copy_from_slice(&rows.to_ne_bytes()); + raw[4..6].copy_from_slice(&cols.to_ne_bytes()); + } + } + + raw + } +} -- 2.47.3