[PATCH pve-http-server 1/2] apiserver: add opt-in recording of api requests

[Hannes Laimer <h.laimer@proxmox.com>]

Add a hook in handle_api2_request that, when enabled, records the
requests the server handles (method, path and the full parameters) as
JSON lines in a per-recorder spool under /run/pve-api-record/ for an
external consumer to follow. Unlike the access log this captures the
decoded parameters including request bodies, and runs in pveproxy where
the full request is seen before any proxying.

It is off unless it is enabled via the 'request_recording' flag, all I/O
is wrapped and errors only logged, so this won't interfere with regular
request handling.

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/Makefile                         |   1 +
 src/PVE/APIServer/AnyEvent.pm        |  17 +++
 src/PVE/APIServer/RequestRecorder.pm | 188 +++++++++++++++++++++++++++
 3 files changed, 206 insertions(+)
 create mode 100644 src/PVE/APIServer/RequestRecorder.pm

diff --git a/src/Makefile b/src/Makefile
index 9e1a8f7..e60a8d8 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -8,6 +8,7 @@ install: PVE
 	install -d -m 755 ${PERL5DIR}/PVE/APIServer
 	install -m 0644 PVE/APIServer/AnyEvent.pm ${PERL5DIR}/PVE/APIServer
 	install -m 0644 PVE/APIServer/Formatter.pm ${PERL5DIR}/PVE/APIServer
+	install -m 0644 PVE/APIServer/RequestRecorder.pm ${PERL5DIR}/PVE/APIServer
 	install -m 0644 PVE/APIServer/Utils.pm ${PERL5DIR}/PVE/APIServer
 	install -d -m 755 ${PERL5DIR}/PVE/APIServer/Formatter
 	install -m 0644 PVE/APIServer/Formatter/Standard.pm ${PERL5DIR}/PVE/APIServer/Formatter
diff --git a/src/PVE/APIServer/AnyEvent.pm b/src/PVE/APIServer/AnyEvent.pm
index 915d678..78e96e8 100644
--- a/src/PVE/APIServer/AnyEvent.pm
+++ b/src/PVE/APIServer/AnyEvent.pm
@@ -47,6 +47,7 @@ use PVE::SafeSyslog;
 use PVE::Tools qw(trim);
 
 use PVE::APIServer::Formatter;
+use PVE::APIServer::RequestRecorder;
 use PVE::APIServer::Utils;
 
 my $limit_max_headers = 64;
@@ -977,6 +978,22 @@ sub handle_api2_request {
 
         delete $params->{_dc} if $params; # remove disable cache parameter
 
+        if ($self->{request_recording}) {
+            # opt-in request recording, does not affect normal request handling, all IO is wrapped
+            eval {
+                PVE::APIServer::RequestRecorder::record_event({
+                    method => $method,
+                    path => $rel_uri,
+                    params => $params,
+                    userid => $auth->{userid},
+                    format => $format,
+                    client_ip => $reqstate->{peer_host},
+                    proxied => $r->header('PVEDisableProxy') ? 1 : 0,
+                });
+            };
+            syslog('err', "request recording error: $@") if $@ && $self->{debug};
+        }
+
         my $clientip = $reqstate->{peer_host};
 
         my $res = $self->rest_handler($clientip, $method, $rel_uri, $auth, $params, $format);
diff --git a/src/PVE/APIServer/RequestRecorder.pm b/src/PVE/APIServer/RequestRecorder.pm
new file mode 100644
index 0000000..a951bd7
--- /dev/null
+++ b/src/PVE/APIServer/RequestRecorder.pm
@@ -0,0 +1,188 @@
+package PVE::APIServer::RequestRecorder;
+
+# Request recorder: the API server calls record_event() for every
+# request and, while a recording is active, appends the matching ones as JSON
+# lines to a per-recorder spool in /run/pve-api-record/ for a client to read.
+
+use strict;
+use warnings;
+
+use Fcntl qw(O_RDONLY O_WRONLY O_APPEND O_CREAT O_TRUNC LOCK_EX LOCK_NB);
+use JSON;
+use Sys::Syslog qw(syslog);
+use Time::HiRes qw(time);
+
+use constant RECORD_DIR => '/run/pve-api-record';
+
+# state-changing methods, recorded by default (use 'all' for the rest)
+my $write_methods = { POST => 1, PUT => 1, DELETE => 1 };
+
+sub record_dir { return RECORD_DIR; }
+sub request_path { my ($id) = @_; return RECORD_DIR . "/$id.req"; }
+sub events_path { my ($id) = @_; return RECORD_DIR . "/$id.events"; }
+
+# daemon side (runs in pveproxy, under taint mode)
+
+# per-worker cache so an idle server only checks the spool dir ~once a second
+my $cache = { checked => 0, recorders => [] };
+
+# Returns the live recorders and reaps dead ones in one pass: we try to grab
+# each request file's lock, and getting it means no client holds it, so we
+# delete the files under the lock. A held lock means a live client, whose
+# filter we read fresh each refresh (a reused PID can rewrite its .req).
+# All best-effort, so recording never depends on reaping.
+my $active_recorders = sub {
+    my $now = time();
+    return $cache->{recorders} if ($now - $cache->{checked}) < 1.0;
+    $cache->{checked} = $now;
+
+    my $list = [];
+    my $dh;
+    if (!opendir($dh, RECORD_DIR)) { # no spool dir means nobody is recording
+        $cache->{recorders} = $list;
+        return $list;
+    }
+
+    while (defined(my $entry = readdir($dh))) {
+        next if $entry !~ /^(\d+)\.req$/;
+        my $id = $1;
+
+        eval {
+            sysopen(my $fh, request_path($id), O_RDONLY) or return;
+            if (flock($fh, LOCK_EX | LOCK_NB)) {
+                # got the lock: the client is gone, cleanup under the lock
+                unlink(request_path($id), events_path($id));
+                close($fh);
+            } else {
+                # a live client holds the lock: read its current filter
+                my $raw = do { local $/ = undef; <$fh> };
+                close($fh);
+                my $req = eval { decode_json($raw) };
+                if ($req && ref($req) eq 'HASH') {
+                    $req->{id} = $id;
+                    push @$list, $req;
+                }
+            }
+        };
+    }
+    closedir($dh);
+
+    $cache->{recorders} = $list;
+    return $list;
+};
+
+my $matches = sub {
+    my ($req, $userid, $method) = @_;
+
+    my $want = $req->{userid};
+    if (defined($want) && $want ne '') {
+        # match the user and any of their API tokens ("user@realm!token")
+        my $ok = ($userid eq $want) || (index($userid, "$want!") == 0);
+        return 0 if !$ok;
+    }
+
+    return 0 if !$req->{all} && !$write_methods->{$method};
+
+    return 1;
+};
+
+my $append_event = sub {
+    my ($id, $event) = @_;
+
+    my $line = eval { encode_json($event) };
+    return undef if !defined($line);
+
+    # one O_APPEND line is atomic on tmpfs, so concurrent workers don't interleave
+    sysopen(my $fh, events_path($id), O_WRONLY | O_APPEND | O_CREAT, 0640) or return undef;
+    my $rc = syswrite($fh, "$line\n");
+    close($fh);
+
+    return defined($rc) ? 1 : undef;
+};
+
+# called for every request. $event has method, path, params, userid and
+# optionally proxied/format/client_ip. never dies, never blocks.
+sub record_event {
+    my ($event) = @_;
+
+    eval {
+        my $recorders = $active_recorders->();
+        return if !@$recorders;
+        return if $event->{proxied}; # skip internal inter-node proxy hops
+
+        my $userid = $event->{userid} // '';
+        my $method = $event->{method} // '';
+
+        for my $req (@$recorders) {
+            next if !$matches->($req, $userid, $method);
+            $append_event->($req->{id}, $event);
+        }
+    };
+    syslog('err', "request-recorder: record_event error: $@") if $@;
+
+    return;
+}
+
+# client side (runs as root, starts/stops a recording)
+
+# setgid root:www-data so the www-data daemon can drop event files beside
+# the root-written request file
+sub ensure_record_dir {
+    my $dir = RECORD_DIR;
+
+    if (!-d $dir) {
+        mkdir($dir) or die "unable to create spool directory '$dir': $!\n";
+    }
+
+    if (defined(my $gid = getgrnam('www-data'))) {
+        chown(0, $gid, $dir);
+    }
+    chmod(02770, $dir);
+
+    return $dir;
+}
+
+# kept open for our whole lifetime, so the daemon's reaper sees us as alive
+my $lock_fh;
+
+sub start_recording {
+    my ($filter) = @_; # { userid => ..., all => 0|1 }
+
+    ensure_record_dir();
+
+    my $id = $$;
+    my $req = {
+        userid => $filter->{userid},
+        all => $filter->{all} ? 1 : 0,
+    };
+
+    unlink(events_path($id));
+
+    my $path = request_path($id);
+    sysopen(my $fh, $path, O_WRONLY | O_CREAT | O_TRUNC, 0640)
+        or die "unable to write recorder request '$path': $!\n";
+    flock($fh, LOCK_EX | LOCK_NB)
+        or die "unable to lock recorder request '$path': $!\n";
+    syswrite($fh, encode_json($req));
+
+    # group-readable so the www-data daemon can read the filter and test the lock
+    if (defined(my $gid = getgrnam('www-data'))) {
+        chown(0, $gid, $path);
+    }
+
+    $lock_fh = $fh;
+
+    return $id;
+}
+
+sub stop_recording {
+    my ($id) = @_;
+    if ($lock_fh) {
+        close($lock_fh);
+        undef $lock_fh;
+    }
+    unlink(request_path($id), events_path($id));
+    return;
+}
+
+1;
-- 
2.47.3