From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id E823B1FF14F for ; Wed, 17 Jun 2026 18:48:27 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id AAF302F18; Wed, 17 Jun 2026 18:48:27 +0200 (CEST) From: Hannes Laimer To: pve-devel@lists.proxmox.com Subject: [PATCH pve-http-server 1/2] apiserver: add opt-in recording of api requests Date: Wed, 17 Jun 2026 18:47:48 +0200 Message-ID: <20260617164749.574759-2-h.laimer@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260617164749.574759-1-h.laimer@proxmox.com> References: <20260617164749.574759-1-h.laimer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1781714815075 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.914 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_MAILER 2 Automated Mailer Tag Left in Email SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [formatter.pm,utils.pm,requestrecorder.pm,anyevent.pm,id.events,standard.pm] Message-ID-Hash: BNCJBATMSLKZUPZ5LMZCYEEEZ562UJEB X-Message-ID-Hash: BNCJBATMSLKZUPZ5LMZCYEEEZ562UJEB X-MailFrom: h.laimer@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add a hook in handle_api2_request that, when enabled, records the requests the server handles (method, path and the full parameters) as JSON lines in a per-recorder spool under /run/pve-api-record/ for an external consumer to follow. Unlike the access log this captures the decoded parameters including request bodies, and runs in pveproxy where the full request is seen before any proxying. It is off unless it is enabled via the 'request_recording' flag, all I/O is wrapped and errors only logged, so this won't interfere with regular request handling. Signed-off-by: Hannes Laimer --- src/Makefile | 1 + src/PVE/APIServer/AnyEvent.pm | 17 +++ src/PVE/APIServer/RequestRecorder.pm | 188 +++++++++++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 src/PVE/APIServer/RequestRecorder.pm diff --git a/src/Makefile b/src/Makefile index 9e1a8f7..e60a8d8 100644 --- a/src/Makefile +++ b/src/Makefile @@ -8,6 +8,7 @@ install: PVE install -d -m 755 ${PERL5DIR}/PVE/APIServer install -m 0644 PVE/APIServer/AnyEvent.pm ${PERL5DIR}/PVE/APIServer install -m 0644 PVE/APIServer/Formatter.pm ${PERL5DIR}/PVE/APIServer + install -m 0644 PVE/APIServer/RequestRecorder.pm ${PERL5DIR}/PVE/APIServer install -m 0644 PVE/APIServer/Utils.pm ${PERL5DIR}/PVE/APIServer install -d -m 755 ${PERL5DIR}/PVE/APIServer/Formatter install -m 0644 PVE/APIServer/Formatter/Standard.pm ${PERL5DIR}/PVE/APIServer/Formatter diff --git a/src/PVE/APIServer/AnyEvent.pm b/src/PVE/APIServer/AnyEvent.pm index 915d678..78e96e8 100644 --- a/src/PVE/APIServer/AnyEvent.pm +++ b/src/PVE/APIServer/AnyEvent.pm @@ -47,6 +47,7 @@ use PVE::SafeSyslog; use PVE::Tools qw(trim); use PVE::APIServer::Formatter; +use PVE::APIServer::RequestRecorder; use PVE::APIServer::Utils; my $limit_max_headers = 64; @@ -977,6 +978,22 @@ sub handle_api2_request { delete $params->{_dc} if $params; # remove disable cache parameter + if ($self->{request_recording}) { + # opt-in request recording, does not affect normal request handling, all IO is wrapped + eval { + PVE::APIServer::RequestRecorder::record_event({ + method => $method, + path => $rel_uri, + params => $params, + userid => $auth->{userid}, + format => $format, + client_ip => $reqstate->{peer_host}, + proxied => $r->header('PVEDisableProxy') ? 1 : 0, + }); + }; + syslog('err', "request recording error: $@") if $@ && $self->{debug}; + } + my $clientip = $reqstate->{peer_host}; my $res = $self->rest_handler($clientip, $method, $rel_uri, $auth, $params, $format); diff --git a/src/PVE/APIServer/RequestRecorder.pm b/src/PVE/APIServer/RequestRecorder.pm new file mode 100644 index 0000000..a951bd7 --- /dev/null +++ b/src/PVE/APIServer/RequestRecorder.pm @@ -0,0 +1,188 @@ +package PVE::APIServer::RequestRecorder; + +# Request recorder: the API server calls record_event() for every +# request and, while a recording is active, appends the matching ones as JSON +# lines to a per-recorder spool in /run/pve-api-record/ for a client to read. + +use strict; +use warnings; + +use Fcntl qw(O_RDONLY O_WRONLY O_APPEND O_CREAT O_TRUNC LOCK_EX LOCK_NB); +use JSON; +use Sys::Syslog qw(syslog); +use Time::HiRes qw(time); + +use constant RECORD_DIR => '/run/pve-api-record'; + +# state-changing methods, recorded by default (use 'all' for the rest) +my $write_methods = { POST => 1, PUT => 1, DELETE => 1 }; + +sub record_dir { return RECORD_DIR; } +sub request_path { my ($id) = @_; return RECORD_DIR . "/$id.req"; } +sub events_path { my ($id) = @_; return RECORD_DIR . "/$id.events"; } + +# daemon side (runs in pveproxy, under taint mode) + +# per-worker cache so an idle server only checks the spool dir ~once a second +my $cache = { checked => 0, recorders => [] }; + +# Returns the live recorders and reaps dead ones in one pass: we try to grab +# each request file's lock, and getting it means no client holds it, so we +# delete the files under the lock. A held lock means a live client, whose +# filter we read fresh each refresh (a reused PID can rewrite its .req). +# All best-effort, so recording never depends on reaping. +my $active_recorders = sub { + my $now = time(); + return $cache->{recorders} if ($now - $cache->{checked}) < 1.0; + $cache->{checked} = $now; + + my $list = []; + my $dh; + if (!opendir($dh, RECORD_DIR)) { # no spool dir means nobody is recording + $cache->{recorders} = $list; + return $list; + } + + while (defined(my $entry = readdir($dh))) { + next if $entry !~ /^(\d+)\.req$/; + my $id = $1; + + eval { + sysopen(my $fh, request_path($id), O_RDONLY) or return; + if (flock($fh, LOCK_EX | LOCK_NB)) { + # got the lock: the client is gone, cleanup under the lock + unlink(request_path($id), events_path($id)); + close($fh); + } else { + # a live client holds the lock: read its current filter + my $raw = do { local $/ = undef; <$fh> }; + close($fh); + my $req = eval { decode_json($raw) }; + if ($req && ref($req) eq 'HASH') { + $req->{id} = $id; + push @$list, $req; + } + } + }; + } + closedir($dh); + + $cache->{recorders} = $list; + return $list; +}; + +my $matches = sub { + my ($req, $userid, $method) = @_; + + my $want = $req->{userid}; + if (defined($want) && $want ne '') { + # match the user and any of their API tokens ("user@realm!token") + my $ok = ($userid eq $want) || (index($userid, "$want!") == 0); + return 0 if !$ok; + } + + return 0 if !$req->{all} && !$write_methods->{$method}; + + return 1; +}; + +my $append_event = sub { + my ($id, $event) = @_; + + my $line = eval { encode_json($event) }; + return undef if !defined($line); + + # one O_APPEND line is atomic on tmpfs, so concurrent workers don't interleave + sysopen(my $fh, events_path($id), O_WRONLY | O_APPEND | O_CREAT, 0640) or return undef; + my $rc = syswrite($fh, "$line\n"); + close($fh); + + return defined($rc) ? 1 : undef; +}; + +# called for every request. $event has method, path, params, userid and +# optionally proxied/format/client_ip. never dies, never blocks. +sub record_event { + my ($event) = @_; + + eval { + my $recorders = $active_recorders->(); + return if !@$recorders; + return if $event->{proxied}; # skip internal inter-node proxy hops + + my $userid = $event->{userid} // ''; + my $method = $event->{method} // ''; + + for my $req (@$recorders) { + next if !$matches->($req, $userid, $method); + $append_event->($req->{id}, $event); + } + }; + syslog('err', "request-recorder: record_event error: $@") if $@; + + return; +} + +# client side (runs as root, starts/stops a recording) + +# setgid root:www-data so the www-data daemon can drop event files beside +# the root-written request file +sub ensure_record_dir { + my $dir = RECORD_DIR; + + if (!-d $dir) { + mkdir($dir) or die "unable to create spool directory '$dir': $!\n"; + } + + if (defined(my $gid = getgrnam('www-data'))) { + chown(0, $gid, $dir); + } + chmod(02770, $dir); + + return $dir; +} + +# kept open for our whole lifetime, so the daemon's reaper sees us as alive +my $lock_fh; + +sub start_recording { + my ($filter) = @_; # { userid => ..., all => 0|1 } + + ensure_record_dir(); + + my $id = $$; + my $req = { + userid => $filter->{userid}, + all => $filter->{all} ? 1 : 0, + }; + + unlink(events_path($id)); + + my $path = request_path($id); + sysopen(my $fh, $path, O_WRONLY | O_CREAT | O_TRUNC, 0640) + or die "unable to write recorder request '$path': $!\n"; + flock($fh, LOCK_EX | LOCK_NB) + or die "unable to lock recorder request '$path': $!\n"; + syswrite($fh, encode_json($req)); + + # group-readable so the www-data daemon can read the filter and test the lock + if (defined(my $gid = getgrnam('www-data'))) { + chown(0, $gid, $path); + } + + $lock_fh = $fh; + + return $id; +} + +sub stop_recording { + my ($id) = @_; + if ($lock_fh) { + close($lock_fh); + undef $lock_fh; + } + unlink(request_path($id), events_path($id)); + return; +} + +1; -- 2.47.3