[pve-devel] [PATCH pve-storage 1/4] pvestord: setup new pvestord daemon

* [pve-devel] [PATCH pve-storage 1/4] pvestord: setup new pvestord daemon
       [not found] <20251017112539.26471-1-joao.sousa@eurotux.com>
@ 2025-10-17 11:25 ` Tiago Sousa via pve-devel
  2025-10-17 11:25 ` [pve-devel] [PATCH pve-storage 2/4] storage: add extend queue handling Tiago Sousa via pve-devel
                   ` (8 subsequent siblings)
  9 siblings, 0 replies; 10+ messages in thread
From: Tiago Sousa via pve-devel @ 2025-10-17 11:25 UTC (permalink / raw)
  To: pve-devel; +Cc: Tiago Sousa

[-- Attachment #1: Type: message/rfc822, Size: 13672 bytes --]

From: Tiago Sousa <joao.sousa@eurotux.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 1/4] pvestord: setup new pvestord daemon
Date: Fri, 17 Oct 2025 12:25:26 +0100
Message-ID: <20251017112539.26471-2-joao.sousa@eurotux.com>

Signed-off-by: Tiago Sousa <joao.sousa@eurotux.com>
---
 src/Makefile                  |   1 +
 src/PVE/Makefile              |   1 +
 src/PVE/Service/Makefile      |  10 ++
 src/PVE/Service/pvestord.pm   | 193 ++++++++++++++++++++++++++++++++++
 src/bin/Makefile              |   3 +
 src/bin/pvestord              |  24 +++++
 src/services/Makefile         |  14 +++
 src/services/pvestord.service |  15 +++
 8 files changed, 261 insertions(+)
 create mode 100644 src/PVE/Service/Makefile
 create mode 100644 src/PVE/Service/pvestord.pm
 create mode 100755 src/bin/pvestord
 create mode 100644 src/services/Makefile
 create mode 100644 src/services/pvestord.service

diff --git a/src/Makefile b/src/Makefile
index a322f46..09777f2 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -9,6 +9,7 @@ all:
 install: PVE bin udev-rbd
 	$(MAKE) -C bin install
 	$(MAKE) -C PVE install
+	$(MAKE) -C services install
 	$(MAKE) -C udev-rbd install
 
 .PHONY: test
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index 9e9f6aa..01ba360 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -11,6 +11,7 @@ install:
 	make -C API2 install
 	make -C BackupProvider install
 	make -C CLI install
+	make -C Service install
 
 .PHONY: test
 test:
diff --git a/src/PVE/Service/Makefile b/src/PVE/Service/Makefile
new file mode 100644
index 0000000..a581f43
--- /dev/null
+++ b/src/PVE/Service/Makefile
@@ -0,0 +1,10 @@
+SOURCES=pvestord.pm
+
+all:
+
+.PHONY: install
+install: $(SOURCES)
+	install -d -m 0755 $(DESTDIR)$(PERLDIR)/PVE/Service
+	for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/Service/$$i; done
+
+clean:
diff --git a/src/PVE/Service/pvestord.pm b/src/PVE/Service/pvestord.pm
new file mode 100644
index 0000000..29fe016
--- /dev/null
+++ b/src/PVE/Service/pvestord.pm
@@ -0,0 +1,193 @@
+package PVE::Service::pvestord;
+
+use strict;
+use warnings;
+
+use Time::HiRes qw (gettimeofday);
+use PVE::SafeSyslog;
+use PVE::Daemon;
+use PVE::Cluster qw(cfs_read_file);
+use PVE::Storage;
+use PVE::QemuConfig;
+use PVE::QemuServer;
+use PVE::QemuServer::Drive;
+use PVE::QemuServer::Blockdev;
+use PVE::QemuServer::Helpers;
+use PVE::INotify;
+
+use base qw(PVE::Daemon);
+
+my $cmdline = [$0, @ARGV];
+
+my %daemon_options = (restart_on_error => 5, stop_wait_time => 15);
+my $daemon = __PACKAGE__->new('pvestord', $cmdline, %daemon_options);
+
+my $nodename = PVE::INotify::nodename();
+
+sub init {
+    my ($self) = @_;
+    PVE::Cluster::cfs_update();
+}
+
+my sub get_drive_id {
+    my ($block_stats, $blockdev_nodename) = @_;
+    foreach my $drive_id (keys %$block_stats) {
+        my $entry = $block_stats->{$drive_id};
+        my $file_blockdev = $entry->{parent}->{parent};
+        return $drive_id
+            if ($file_blockdev->{'node-name'} eq $blockdev_nodename);
+    }
+    return undef;
+}
+
+my sub dequeue {
+    my ($queue) = @_;
+    PVE::Storage::lock_extend_queue(
+        sub {
+            # TODO: This will have to have some sort of mechanism
+            # to make sure that the element that is removed is the one
+            # that this node is handling
+            shift @$queue;
+
+            PVE::Storage::write_extend_queue($queue);
+        },
+        "Could not lock extend queue file",
+    );
+}
+
+sub perform_extend {
+    my $storecfg = PVE::Storage::config();
+    my $queue = PVE::Storage::extend_queue();
+
+    my $first_extend_request = @$queue[0];
+    return if !$first_extend_request;
+
+    my ($vmid, $blockdev_nodename) = @$first_extend_request;
+
+    my $vmlist = PVE::Cluster::get_vmlist();
+    my $owner_nodename = $vmlist->{ids}->{$vmid}->{node};
+
+    if ($owner_nodename eq $nodename) {
+        my $running = PVE::QemuServer::Helpers::vm_running_locally($vmid);
+        # NOTE: The block device node name is currently generated using a SHA-256 hash,
+        # which makes it impossible to reverse-engineer and identify the original disk.
+        # As a result, we must rely on `blockstats` to determine which disk corresponds
+        # to a given node name — but these statistics are only available when the machine is running.
+        # Consider updating the `get_node_name()` function to use a reversible encoding
+        # (e.g., Base64) instead of a SHA-256 digest to simplify disk identification.
+
+        my $extend_function = sub {
+            dequeue($queue);
+            syslog("info", "Processsing extend request $vmid: $blockdev_nodename\n");
+
+            my $block_stats = PVE::QemuServer::Blockdev::get_block_stats($vmid);
+
+            my $drive_id = get_drive_id($block_stats, $blockdev_nodename);
+            if (!$drive_id) {
+                syslog("err", "Couldn't find drive_id for blockdev $blockdev_nodename");
+                return;
+            }
+            my $vm_conf = PVE::QemuConfig->load_config($vmid);
+            my $drive = PVE::QemuServer::parse_drive($drive_id, $vm_conf->{$drive_id});
+            my $volid = $drive->{file};
+
+            PVE::QemuServer::Blockdev::underlay_resize(
+                $storecfg, $vmid, $drive_id, $volid
+            );
+        };
+        PVE::QemuConfig->lock_config($vmid, $extend_function);
+    }
+}
+
+my $next_update = 0;
+my $cycle = 0;
+my $restart_request = 0;
+
+my $initial_memory_usage = 0;
+
+# 1 second cycles
+my $updatetime = 1;
+
+sub run {
+    my ($self) = @_;
+    syslog("info", "Running on node $nodename\n");
+
+    for (;;) { # forever
+        # get next extend request
+        $next_update = time() + $updatetime;
+
+        if ($cycle) {
+            my ($ccsec, $cusec) = gettimeofday();
+            eval {
+                # syslog('info', "start status update");
+                PVE::Cluster::cfs_update();
+                perform_extend();
+            };
+            my $err = $@;
+
+            if ($err) {
+                syslog('err', "status update error: $err");
+            }
+
+            my ($ccsec_end, $cusec_end) = gettimeofday();
+            my $cptime = ($ccsec_end - $ccsec) + ($cusec_end - $cusec) / 1000000;
+
+            syslog('info', sprintf("extend process time (%.3f seconds)", $cptime))
+                if ($cptime > 1);
+        }
+
+        $cycle++;
+
+        my $mem = PVE::ProcFSTools::read_memory_usage();
+        my $resident_kb = $mem->{resident} / 1024;
+
+        if (!defined($initial_memory_usage) || ($cycle < 10)) {
+            $initial_memory_usage = $resident_kb;
+        } else {
+            my $diff = $resident_kb - $initial_memory_usage;
+            if ($diff > 15 * 1024) {
+                syslog(
+                    'info',
+                    "restarting server after $cycle cycles to "
+                        . "reduce memory usage (free $resident_kb ($diff) KB)",
+                );
+                $self->restart_daemon();
+            }
+        }
+
+        my $wcount = 0;
+        while (
+            (time() < $next_update)
+            && ($wcount < $updatetime)
+            && # protect against time wrap
+            !$restart_request
+        ) {
+            $wcount++;
+            sleep(1);
+        }
+
+        $self->restart_daemon() if $restart_request;
+    }
+}
+
+sub shutdown {
+    my ($self) = @_;
+
+    syslog('info', "server closing");
+
+    $self->exit_daemon(0);
+}
+
+$daemon->register_start_command();
+$daemon->register_restart_command(1);
+$daemon->register_stop_command();
+$daemon->register_status_command();
+
+our $cmddef = {
+    start => [__PACKAGE__, 'start', []],
+    restart => [__PACKAGE__, 'restart', []],
+    stop => [__PACKAGE__, 'stop', []],
+    status => [__PACKAGE__, 'status', [], undef, sub { print shift . "\n"; }],
+};
+
+1;
diff --git a/src/bin/Makefile b/src/bin/Makefile
index 2e0a080..206c35b 100644
--- a/src/bin/Makefile
+++ b/src/bin/Makefile
@@ -1,5 +1,6 @@
 DESTDIR=
 PREFIX=/usr
+BINDIR=$(PREFIX)/bin
 SBINDIR=$(PREFIX)/sbin
 MANDIR=$(PREFIX)/share/man
 MAN1DIR=$(MANDIR)/man1/
@@ -30,6 +31,8 @@ install: pvesm.1 pvesm.bash-completion pvesm.zsh-completion
 	gzip -9 -n $(DESTDIR)$(MAN1DIR)/pvesm.1
 	install -m 0644 -D pvesm.bash-completion $(DESTDIR)$(BASHCOMPLDIR)/pvesm
 	install -m 0644 -D pvesm.zsh-completion $(DESTDIR)$(ZSHCOMPLDIR)/_pvesm
+	install -d $(DESTDIR)$(BINDIR)
+	install -m 0755 pvestord $(DESTDIR)$(BINDIR)
 
 .PHONY: clean
 clean:
diff --git a/src/bin/pvestord b/src/bin/pvestord
new file mode 100755
index 0000000..e88a0b3
--- /dev/null
+++ b/src/bin/pvestord
@@ -0,0 +1,24 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use PVE::INotify;
+use PVE::RPCEnvironment;
+use PVE::SafeSyslog;
+use PVE::Service::pvestord;
+
+$SIG{'__WARN__'} = sub {
+    my $err = $@;
+    my $t = $_[0];
+    chomp $t;
+    print STDERR "$t\n";
+    syslog('warning', "%s", $t);
+    $@ = $err;
+};
+
+my $prepare = sub {
+
+};
+
+PVE::Service::pvestord->run_cli_handler(prepare => $prepare);
diff --git a/src/services/Makefile b/src/services/Makefile
new file mode 100644
index 0000000..98db674
--- /dev/null
+++ b/src/services/Makefile
@@ -0,0 +1,14 @@
+SERVICEDIR=$(DESTDIR)/usr/lib/systemd/system
+
+all:
+
+SERVICES= pvestord.service
+
+.PHONY: install
+install: $(SERVICES)
+	install -d $(SERVICEDIR)
+	install -m 0644 $(SERVICES) $(SERVICEDIR)
+
+.PHONY: clean
+clean:
+	rm -rf *~
diff --git a/src/services/pvestord.service b/src/services/pvestord.service
new file mode 100644
index 0000000..310fa91
--- /dev/null
+++ b/src/services/pvestord.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=PVE Storage Monitor Daemon
+ConditionPathExists=/usr/bin/pvestord
+Wants=pve-cluster.service
+After=pve-cluster.service
+
+[Service]
+ExecStart=/usr/bin/pvestord start
+ExecStop=/usr/bin/pvestord stop
+ExecReload=/usr/bin/pvestord restart
+PIDFile=/run/pvestord.pid
+Type=forking
+
+[Install]
+WantedBy=multi-user.target
-- 
2.47.3



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

^ permalink raw reply	[flat|nested] 10+ messages in thread