* [PATCH pve-http-server 1/2] apiserver: add opt-in recording of api requests
2026-06-17 16:47 [RFC http-server/manager 0/2] add pvesh record subcommand Hannes Laimer
@ 2026-06-17 16:47 ` Hannes Laimer
2026-06-17 16:47 ` [PATCH pve-manager 2/2] pvesh: add 'record' subcommand to trace user's API requests Hannes Laimer
1 sibling, 0 replies; 3+ messages in thread
From: Hannes Laimer @ 2026-06-17 16:47 UTC (permalink / raw)
To: pve-devel
Add a hook in handle_api2_request that, when enabled, records the
requests the server handles (method, path and the full parameters) as
JSON lines in a per-recorder spool under /run/pve-api-record/ for an
external consumer to follow. Unlike the access log this captures the
decoded parameters including request bodies, and runs in pveproxy where
the full request is seen before any proxying.
It is off unless it is enabled via the 'request_recording' flag, all I/O
is wrapped and errors only logged, so this won't interfere with regular
request handling.
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
src/Makefile | 1 +
src/PVE/APIServer/AnyEvent.pm | 17 +++
src/PVE/APIServer/RequestRecorder.pm | 188 +++++++++++++++++++++++++++
3 files changed, 206 insertions(+)
create mode 100644 src/PVE/APIServer/RequestRecorder.pm
diff --git a/src/Makefile b/src/Makefile
index 9e1a8f7..e60a8d8 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -8,6 +8,7 @@ install: PVE
install -d -m 755 ${PERL5DIR}/PVE/APIServer
install -m 0644 PVE/APIServer/AnyEvent.pm ${PERL5DIR}/PVE/APIServer
install -m 0644 PVE/APIServer/Formatter.pm ${PERL5DIR}/PVE/APIServer
+ install -m 0644 PVE/APIServer/RequestRecorder.pm ${PERL5DIR}/PVE/APIServer
install -m 0644 PVE/APIServer/Utils.pm ${PERL5DIR}/PVE/APIServer
install -d -m 755 ${PERL5DIR}/PVE/APIServer/Formatter
install -m 0644 PVE/APIServer/Formatter/Standard.pm ${PERL5DIR}/PVE/APIServer/Formatter
diff --git a/src/PVE/APIServer/AnyEvent.pm b/src/PVE/APIServer/AnyEvent.pm
index 915d678..78e96e8 100644
--- a/src/PVE/APIServer/AnyEvent.pm
+++ b/src/PVE/APIServer/AnyEvent.pm
@@ -47,6 +47,7 @@ use PVE::SafeSyslog;
use PVE::Tools qw(trim);
use PVE::APIServer::Formatter;
+use PVE::APIServer::RequestRecorder;
use PVE::APIServer::Utils;
my $limit_max_headers = 64;
@@ -977,6 +978,22 @@ sub handle_api2_request {
delete $params->{_dc} if $params; # remove disable cache parameter
+ if ($self->{request_recording}) {
+ # opt-in request recording, does not affect normal request handling, all IO is wrapped
+ eval {
+ PVE::APIServer::RequestRecorder::record_event({
+ method => $method,
+ path => $rel_uri,
+ params => $params,
+ userid => $auth->{userid},
+ format => $format,
+ client_ip => $reqstate->{peer_host},
+ proxied => $r->header('PVEDisableProxy') ? 1 : 0,
+ });
+ };
+ syslog('err', "request recording error: $@") if $@ && $self->{debug};
+ }
+
my $clientip = $reqstate->{peer_host};
my $res = $self->rest_handler($clientip, $method, $rel_uri, $auth, $params, $format);
diff --git a/src/PVE/APIServer/RequestRecorder.pm b/src/PVE/APIServer/RequestRecorder.pm
new file mode 100644
index 0000000..a951bd7
--- /dev/null
+++ b/src/PVE/APIServer/RequestRecorder.pm
@@ -0,0 +1,188 @@
+package PVE::APIServer::RequestRecorder;
+
+# Request recorder: the API server calls record_event() for every
+# request and, while a recording is active, appends the matching ones as JSON
+# lines to a per-recorder spool in /run/pve-api-record/ for a client to read.
+
+use strict;
+use warnings;
+
+use Fcntl qw(O_RDONLY O_WRONLY O_APPEND O_CREAT O_TRUNC LOCK_EX LOCK_NB);
+use JSON;
+use Sys::Syslog qw(syslog);
+use Time::HiRes qw(time);
+
+use constant RECORD_DIR => '/run/pve-api-record';
+
+# state-changing methods, recorded by default (use 'all' for the rest)
+my $write_methods = { POST => 1, PUT => 1, DELETE => 1 };
+
+sub record_dir { return RECORD_DIR; }
+sub request_path { my ($id) = @_; return RECORD_DIR . "/$id.req"; }
+sub events_path { my ($id) = @_; return RECORD_DIR . "/$id.events"; }
+
+# daemon side (runs in pveproxy, under taint mode)
+
+# per-worker cache so an idle server only checks the spool dir ~once a second
+my $cache = { checked => 0, recorders => [] };
+
+# Returns the live recorders and reaps dead ones in one pass: we try to grab
+# each request file's lock, and getting it means no client holds it, so we
+# delete the files under the lock. A held lock means a live client, whose
+# filter we read fresh each refresh (a reused PID can rewrite its .req).
+# All best-effort, so recording never depends on reaping.
+my $active_recorders = sub {
+ my $now = time();
+ return $cache->{recorders} if ($now - $cache->{checked}) < 1.0;
+ $cache->{checked} = $now;
+
+ my $list = [];
+ my $dh;
+ if (!opendir($dh, RECORD_DIR)) { # no spool dir means nobody is recording
+ $cache->{recorders} = $list;
+ return $list;
+ }
+
+ while (defined(my $entry = readdir($dh))) {
+ next if $entry !~ /^(\d+)\.req$/;
+ my $id = $1;
+
+ eval {
+ sysopen(my $fh, request_path($id), O_RDONLY) or return;
+ if (flock($fh, LOCK_EX | LOCK_NB)) {
+ # got the lock: the client is gone, cleanup under the lock
+ unlink(request_path($id), events_path($id));
+ close($fh);
+ } else {
+ # a live client holds the lock: read its current filter
+ my $raw = do { local $/ = undef; <$fh> };
+ close($fh);
+ my $req = eval { decode_json($raw) };
+ if ($req && ref($req) eq 'HASH') {
+ $req->{id} = $id;
+ push @$list, $req;
+ }
+ }
+ };
+ }
+ closedir($dh);
+
+ $cache->{recorders} = $list;
+ return $list;
+};
+
+my $matches = sub {
+ my ($req, $userid, $method) = @_;
+
+ my $want = $req->{userid};
+ if (defined($want) && $want ne '') {
+ # match the user and any of their API tokens ("user@realm!token")
+ my $ok = ($userid eq $want) || (index($userid, "$want!") == 0);
+ return 0 if !$ok;
+ }
+
+ return 0 if !$req->{all} && !$write_methods->{$method};
+
+ return 1;
+};
+
+my $append_event = sub {
+ my ($id, $event) = @_;
+
+ my $line = eval { encode_json($event) };
+ return undef if !defined($line);
+
+ # one O_APPEND line is atomic on tmpfs, so concurrent workers don't interleave
+ sysopen(my $fh, events_path($id), O_WRONLY | O_APPEND | O_CREAT, 0640) or return undef;
+ my $rc = syswrite($fh, "$line\n");
+ close($fh);
+
+ return defined($rc) ? 1 : undef;
+};
+
+# called for every request. $event has method, path, params, userid and
+# optionally proxied/format/client_ip. never dies, never blocks.
+sub record_event {
+ my ($event) = @_;
+
+ eval {
+ my $recorders = $active_recorders->();
+ return if !@$recorders;
+ return if $event->{proxied}; # skip internal inter-node proxy hops
+
+ my $userid = $event->{userid} // '';
+ my $method = $event->{method} // '';
+
+ for my $req (@$recorders) {
+ next if !$matches->($req, $userid, $method);
+ $append_event->($req->{id}, $event);
+ }
+ };
+ syslog('err', "request-recorder: record_event error: $@") if $@;
+
+ return;
+}
+
+# client side (runs as root, starts/stops a recording)
+
+# setgid root:www-data so the www-data daemon can drop event files beside
+# the root-written request file
+sub ensure_record_dir {
+ my $dir = RECORD_DIR;
+
+ if (!-d $dir) {
+ mkdir($dir) or die "unable to create spool directory '$dir': $!\n";
+ }
+
+ if (defined(my $gid = getgrnam('www-data'))) {
+ chown(0, $gid, $dir);
+ }
+ chmod(02770, $dir);
+
+ return $dir;
+}
+
+# kept open for our whole lifetime, so the daemon's reaper sees us as alive
+my $lock_fh;
+
+sub start_recording {
+ my ($filter) = @_; # { userid => ..., all => 0|1 }
+
+ ensure_record_dir();
+
+ my $id = $$;
+ my $req = {
+ userid => $filter->{userid},
+ all => $filter->{all} ? 1 : 0,
+ };
+
+ unlink(events_path($id));
+
+ my $path = request_path($id);
+ sysopen(my $fh, $path, O_WRONLY | O_CREAT | O_TRUNC, 0640)
+ or die "unable to write recorder request '$path': $!\n";
+ flock($fh, LOCK_EX | LOCK_NB)
+ or die "unable to lock recorder request '$path': $!\n";
+ syswrite($fh, encode_json($req));
+
+ # group-readable so the www-data daemon can read the filter and test the lock
+ if (defined(my $gid = getgrnam('www-data'))) {
+ chown(0, $gid, $path);
+ }
+
+ $lock_fh = $fh;
+
+ return $id;
+}
+
+sub stop_recording {
+ my ($id) = @_;
+ if ($lock_fh) {
+ close($lock_fh);
+ undef $lock_fh;
+ }
+ unlink(request_path($id), events_path($id));
+ return;
+}
+
+1;
--
2.47.3
^ permalink raw reply related [flat|nested] 3+ messages in thread* [PATCH pve-manager 2/2] pvesh: add 'record' subcommand to trace user's API requests
2026-06-17 16:47 [RFC http-server/manager 0/2] add pvesh record subcommand Hannes Laimer
2026-06-17 16:47 ` [PATCH pve-http-server 1/2] apiserver: add opt-in recording of api requests Hannes Laimer
@ 2026-06-17 16:47 ` Hannes Laimer
1 sibling, 0 replies; 3+ messages in thread
From: Hannes Laimer @ 2026-06-17 16:47 UTC (permalink / raw)
To: pve-devel
Given a user, `pvesh record <user>` records the api requests made by
that user and translates them into their corresponding `pvesh` command.
By default only writes (POST/PUT/DELETE) are recorded, with '--all' also
reads (GET). Requests are recorded and printed as long as the command
runs.
Makes it easy to repeat things done through the webUI via a script, for
example.
Can only be run as root, like the rest of pvesh.
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
PVE/CLI/pvesh.pm | 140 ++++++++++++++++++++++++++++++++++++++++
PVE/Service/pveproxy.pm | 1 +
2 files changed, 141 insertions(+)
diff --git a/PVE/CLI/pvesh.pm b/PVE/CLI/pvesh.pm
index acd9a605..fa01b01c 100755
--- a/PVE/CLI/pvesh.pm
+++ b/PVE/CLI/pvesh.pm
@@ -12,6 +12,7 @@ use PVE::RPCEnvironment;
use PVE::RESTHandler;
use PVE::CLIFormatter;
use PVE::CLIHandler;
+use PVE::APIServer::RequestRecorder;
use PVE::API2Tools;
use PVE::API2;
use JSON;
@@ -82,6 +83,37 @@ my $method_map = {
delete => 'DELETE',
};
+# reverse of $method_map: HTTP method -> pvesh command
+my $command_map = { reverse %$method_map };
+
+# turn a recorded request event into the equivalent, ready-to-run 'pvesh' command
+my $event_to_pvesh = sub {
+ my ($event) = @_;
+
+ my $method = $event->{method} // return undef;
+ my $cmd = $command_map->{$method} // return undef;
+ my $path = $event->{path} // return undef;
+
+ my $args = ['pvesh', $cmd, $path];
+
+ my $params = $event->{params};
+ if (ref($params) eq 'HASH') {
+ foreach my $key (sort keys %$params) {
+ my $val = $params->{$key};
+ if (ref($val) eq 'ARRAY') {
+ push @$args, "--$key", $_ for @$val;
+ } elsif (ref($val)) {
+ # nested structure from a JSON request body - keep it as JSON
+ push @$args, "--$key", encode_json($val);
+ } else {
+ push @$args, "--$key", $val;
+ }
+ }
+ }
+
+ return String::ShellQuote::shell_quote(@$args);
+};
+
sub check_proxyto {
my ($info, $uri_param, $params) = @_;
@@ -604,6 +636,113 @@ __PACKAGE__->register_method({
},
});
+__PACKAGE__->register_method({
+ name => 'record',
+ path => 'record',
+ method => 'GET',
+ description => "Watch the API requests a user makes (for example via the web UI) and "
+ . "print them as the equivalent 'pvesh' commands. Stop with CTRL+C.",
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ user => get_standard_option(
+ 'userid',
+ {
+ description => "Only record requests made by this user "
+ . "(also matches the user's API tokens).",
+ },
+ ),
+ all => {
+ description => "Also record read-only (GET) requests. By default only "
+ . "writing requests (POST/PUT/DELETE) are recorded.",
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ },
+ },
+ },
+ returns => { type => 'null' },
+ code => sub {
+ my ($param) = @_;
+
+ die "only root can record API requests\n" if $> != 0;
+
+ my $user = $param->{user};
+ my $filter = { userid => $user, all => $param->{all} ? 1 : 0 };
+
+ my $id = PVE::APIServer::RequestRecorder::start_recording($filter);
+
+ my $cleaned = 0;
+ my $cleanup = sub {
+ return if $cleaned;
+ $cleaned = 1;
+ PVE::APIServer::RequestRecorder::stop_recording($id);
+ };
+
+ local $SIG{INT} = local $SIG{TERM} = sub {
+ $cleanup->();
+ print STDERR "\nrecording stopped\n";
+ exit(0);
+ };
+
+ local $| = 1; # show recorded commands as they come in
+
+ my $events_path = PVE::APIServer::RequestRecorder::events_path($id);
+
+ eval {
+ my $fh;
+ my $pos = 0;
+ my $buf = '';
+ my $ino = 0;
+
+ while (1) {
+ # (re)open the spool if it appeared, got rotated to a new inode,
+ # or was truncated/recreated underneath us
+ if (my @st = stat($events_path)) {
+ if (!$fh || $st[1] != $ino || $st[7] < $pos) {
+ open($fh, '<', $events_path)
+ or die "unable to open event spool '$events_path' - $!\n";
+ $ino = $st[1];
+ $pos = 0;
+ $buf = '';
+ }
+ }
+
+ if ($fh) {
+ seek($fh, $pos, 0); # also clears EOF so we pick up appended data
+ my $chunk = do { local $/ = undef; <$fh> };
+ if (defined($chunk) && length($chunk)) {
+ $pos += length($chunk);
+ $buf .= $chunk;
+ while ($buf =~ s/^([^\n]*)\n//) {
+ my $line = $1;
+ next if $line eq '';
+ my $event = eval { decode_json($line) };
+ if (!$event) {
+ next;
+ }
+ my $cmd = $event_to_pvesh->($event);
+ if (!defined($cmd)) {
+ next;
+ }
+ print "$cmd\n";
+ }
+ }
+ }
+
+ select(undef, undef, undef, 0.2);
+ }
+ };
+ my $err = $@;
+
+ $cleanup->();
+ die $err if $err;
+
+ return undef;
+ },
+});
+
our $cmddef = {
usage => [__PACKAGE__, 'usage', ['api_path']],
get => [__PACKAGE__, 'get', ['api_path']],
@@ -611,6 +750,7 @@ our $cmddef = {
set => [__PACKAGE__, 'set', ['api_path']],
create => [__PACKAGE__, 'create', ['api_path']],
delete => [__PACKAGE__, 'delete', ['api_path']],
+ record => [__PACKAGE__, 'record', ['user']],
};
1;
diff --git a/PVE/Service/pveproxy.pm b/PVE/Service/pveproxy.pm
index c6011a00..1dc6fa0f 100755
--- a/PVE/Service/pveproxy.pm
+++ b/PVE/Service/pveproxy.pm
@@ -106,6 +106,7 @@ sub init {
debug => $self->{debug},
trusted_env => 0, # not trusted, anyone can connect
logfile => '/var/log/pveproxy/access.log',
+ request_recording => 1, # enable 'pvesh record' (front-end sees full request bodies)
allow_from => $proxyconf->{ALLOW_FROM},
deny_from => $proxyconf->{DENY_FROM},
policy => $proxyconf->{POLICY},
--
2.47.3
^ permalink raw reply related [flat|nested] 3+ messages in thread