[pve-devel] [PATCH qemu-server v2 13/16] introduce QSD module for qemu-storage-daemon functionality

[Fiona Ebner <f.ebner@proxmox.com>]

For now, supports creating FUSE exports based on Proxmox VE drive
definitions. NBD exports could be added later. In preparation to allow
qcow2 for TPM state volumes. A QEMU storage daemon instance is
associated to a given VM.

Target files where the FUSE export is mounted must already exist. The
'writable' flag for the export is taken to be the negation of the
'read-only' status of the added block node. The 'allow-other' flag is
set to 'off', so only the user the storage daemon is running as may
access the export. For now, exported images don't need to be resized,
so the 'growable' flag is hard-coded to 'false'.

When cleaning up, a 'quit' QMP command is sent to the storage daemon,
with 60 second timeout, after which a SIGTERM is sent with 10 seconds
timeout, before finally SIGKILL is used if the QEMU storage daemon
would still be running.

Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
---

Dependency bump for QEMU needed!

Changes in v2:
* Use ID instead of VM ID for QSD peer.

 src/PVE/QemuServer/Helpers.pm |  33 +++++++++
 src/PVE/QemuServer/Makefile   |   1 +
 src/PVE/QemuServer/QSD.pm     | 130 ++++++++++++++++++++++++++++++++++
 3 files changed, 164 insertions(+)
 create mode 100644 src/PVE/QemuServer/QSD.pm

diff --git a/src/PVE/QemuServer/Helpers.pm b/src/PVE/QemuServer/Helpers.pm
index ce9c352a..35c00754 100644
--- a/src/PVE/QemuServer/Helpers.pm
+++ b/src/PVE/QemuServer/Helpers.pm
@@ -89,6 +89,39 @@ sub qsd_pidfile_name {
     return "${var_run_tmpdir}/qsd-${id}.pid";
 }
 
+sub qsd_fuse_export_cleanup_files {
+    my ($id) = @_;
+
+    # Usually, /var/run is a symlink to /run. It needs to be the exact path for checking if mounted
+    # below. Note that Cwd::realpath() needs to be done on the directory already. Doing it on the
+    # file does not work if the storage daemon is not running and the FUSE is still mounted.
+    my ($real_dir) = Cwd::realpath($var_run_tmpdir) =~ m/^(.*)$/; # untaint
+    if (!$real_dir) {
+        warn "error resolving $var_run_tmpdir - not checking for left-over QSD files\n";
+        return;
+    }
+
+    my $mounts = PVE::ProcFSTools::parse_proc_mounts();
+
+    PVE::Tools::dir_glob_foreach(
+        $real_dir,
+        "qsd-${id}-.*\.fuse",
+        sub {
+            my ($file) = @_;
+            my $path = "${real_dir}/${file}";
+            if (grep { $_->[1] eq $path } $mounts->@*) {
+                PVE::Tools::run_command(['umount', $path]);
+            }
+            unlink $path;
+        },
+    );
+}
+
+sub qsd_fuse_export_path {
+    my ($id, $export_name) = @_;
+    return "${var_run_tmpdir}/qsd-${id}-${export_name}.fuse";
+}
+
 sub vm_pidfile_name {
     my ($vmid) = @_;
     return "${var_run_tmpdir}/$vmid.pid";
diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile
index 63c8d77c..d599ca91 100644
--- a/src/PVE/QemuServer/Makefile
+++ b/src/PVE/QemuServer/Makefile
@@ -23,6 +23,7 @@ SOURCES=Agent.pm	\
 	PCI.pm		\
 	QemuImage.pm	\
 	QMPHelpers.pm	\
+	QSD.pm		\
 	RNG.pm		\
 	RunState.pm	\
 	StateFile.pm	\
diff --git a/src/PVE/QemuServer/QSD.pm b/src/PVE/QemuServer/QSD.pm
new file mode 100644
index 00000000..897ed9cd
--- /dev/null
+++ b/src/PVE/QemuServer/QSD.pm
@@ -0,0 +1,130 @@
+package PVE::QemuServer::QSD;
+
+use v5.36;
+
+use JSON qw(to_json);
+
+use PVE::JSONSchema qw(json_bool);
+use PVE::SafeSyslog qw(syslog);
+use PVE::Storage;
+use PVE::Tools;
+
+use PVE::QemuServer::Blockdev;
+use PVE::QemuServer::Helpers;
+use PVE::QemuServer::Monitor;
+
+=head3 start
+
+    PVE::QemuServer::QSD::start($id);
+
+Start a QEMU storage daemon instance with ID C<$id>.
+
+=cut
+
+sub start($id) {
+    my $name = "QEMU storage daemon $id";
+
+    # If something is still mounted, that could block the new instance, try to clean up first.
+    PVE::QemuServer::Helpers::qsd_fuse_export_cleanup_files($id);
+
+    my $qmp_socket_path =
+        PVE::QemuServer::Helpers::qmp_socket({ name => $name, id => $id, type => 'qsd' });
+    my $pidfile = PVE::QemuServer::Helpers::qsd_pidfile_name($id);
+
+    my $cmd = [
+        'qemu-storage-daemon',
+        '--daemonize',
+        '--chardev',
+        "socket,id=qmp,path=$qmp_socket_path,server=on,wait=off",
+        '--monitor',
+        'chardev=qmp,mode=control',
+        '--pidfile',
+        $pidfile,
+    ];
+
+    PVE::Tools::run_command($cmd);
+
+    my $pid = PVE::QemuServer::Helpers::qsd_running_locally($id);
+    syslog("info", "$name started with PID $pid.");
+
+    return;
+}
+
+=head3 add_fuse_export
+
+    my $path = PVE::QemuServer::QSD::add_fuse_export($id, $drive, $name);
+
+Attach drive C<$drive> to the storage daemon with ID C<$id> and export it with name C<$name> via
+FUSE. Returns the path to the file representing the export.
+
+=cut
+
+sub add_fuse_export($id, $drive, $name) {
+    my $storage_config = PVE::Storage::config();
+
+    PVE::Storage::activate_volumes($storage_config, [$drive->{file}]);
+
+    my ($node_name, $read_only) =
+        PVE::QemuServer::Blockdev::attach($storage_config, $id, $drive, { qsd => 1 });
+
+    my $fuse_path = PVE::QemuServer::Helpers::qsd_fuse_export_path($id, $name);
+    PVE::Tools::file_set_contents($fuse_path, '', 0600); # mountpoint file needs to exist up-front
+
+    my $export = {
+        type => 'fuse',
+        id => "$name",
+        mountpoint => $fuse_path,
+        'node-name' => "$node_name",
+        writable => json_bool(!$read_only),
+        growable => JSON::false,
+        'allow-other' => 'off',
+    };
+
+    PVE::QemuServer::Monitor::qsd_cmd($id, 'block-export-add', $export->%*);
+
+    return $fuse_path;
+}
+
+=head3 quit
+
+    PVE::QemuServer::QSD::quit($id);
+
+Shut down the QEMU storage daemon with ID C<$id> and cleans up its PID file and socket. Waits for 60
+seconds for clean shutdown, then sends SIGTERM and waits an additional 10 seconds before sending
+SIGKILL.
+
+=cut
+
+sub quit($id) {
+    my $name = "QEMU storage daemon $id";
+
+    eval { PVE::QemuServer::Monitor::qsd_cmd($id, 'quit'); };
+    my $qmp_err = $@;
+    warn "$name failed to handle 'quit' - $qmp_err" if $qmp_err;
+
+    my $count = $qmp_err ? 60 : 0; # can't wait for QMP 'quit' to terminate the process if it failed
+    my $pid = PVE::QemuServer::Helpers::qsd_running_locally($id);
+    while ($pid) {
+        if ($count == 60) {
+            warn "$name still running with PID $pid - terminating now with SIGTERM\n";
+            kill 15, $pid;
+        } elsif ($count == 70) {
+            warn "$name still running with PID $pid - terminating now with SIGKILL\n";
+            kill 9, $pid;
+            last;
+        }
+
+        sleep 1;
+        $count++;
+        $pid = PVE::QemuServer::Helpers::qsd_running_locally($id);
+    }
+
+    unlink PVE::QemuServer::Helpers::qsd_pidfile_name($id);
+    unlink PVE::QemuServer::Helpers::qmp_socket({ name => $name, id => $id, type => 'qsd' });
+
+    PVE::QemuServer::Helpers::qsd_fuse_export_cleanup_files($id);
+
+    return;
+}
+
+1;
-- 
2.47.3



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel