From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id C06561FF0C0 for ; Wed, 18 Nov 2026 21:28:41 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id A6284217BC; Wed, 18 Nov 2026 21:27:58 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v7 storage 17/43] diskmanage: read the udev data of all disks with one call Date: Wed, 18 Nov 2026 21:26:00 +0100 Message-ID: <20261118202646.3017286-18-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118202646.3017286-1-t.lamprecht@proxmox.com> References: <20261118202646.3017286-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795033612062 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.017 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 7SZTIRAEXTCXCEBSJ2MCTK47R4DRBPU4 X-Message-ID-Hash: 7SZTIRAEXTCXCEBSJ2MCTK47R4DRBPU4 X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The disk list asks udevadm about each disk on its own, which takes a few milliseconds per disk. That adds up on a node with many disks, for example with a storage plugin that uses one LUN per guest disk. The SAN LUN inventory that follows lists every SAN disk outside of a multipath map. On a lab node with 1000 NVMe/TCP namespaces, it took 6.6 seconds. Read the udev database of all block devices with one call instead. Its entries have the format of the query for a single device, also for a device that udev has not processed yet. On that node, the inventory then took 0.9 seconds. But reading the database gets slower with every block device of a node, also with those that are not listed. On a node with 1000 multipath maps over 2000 iSCSI paths, reading it took 0.47 seconds, while asking about the 14 disks of its list took 0.1 seconds. So the disk list asks about single disks until that took about as long as reading the database would, counting one query as 40 block devices. Signed-off-by: Thomas Lamprecht --- src/PVE/Diskmanage.pm | 63 +++++++++++++++++++++++++++++-------- src/test/disklist_test.pm | 66 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 14 deletions(-) diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm index c4b4ce68..9c16e6a6 100644 --- a/src/PVE/Diskmanage.pm +++ b/src/PVE/Diskmanage.pm @@ -370,21 +370,50 @@ sub get_ceph_volume_infos { return $result; } +# Returns the udev information of all block devices by their kernel name, in the format that +# 'udevadm info --query all' prints for a single device. One call for all devices takes a fraction +# of the time that a call per device takes on a node with many disks. +sub get_udev_db { + my $db = {}; + my $entry = ''; + my $add = sub { + my ($name) = $entry =~ m|^P: .*/([^/\n]+)$|m; + $db->{$name} = $entry if defined($name); + $entry = ''; + }; + eval { + run_command( + ['udevadm', 'info', '--export-db', '--subsystem-match=block'], + outfunc => sub { + my ($line) = @_; + return $add->() if $line eq ''; + $entry .= "$line\n"; + }, + ); + }; + # every entry ends with an empty line, so one that a failed command cut off is left out + warn $@ if $@; + return $db; +} + sub get_udev_info { - my ($dev) = @_; + my ($dev, $udev_db) = @_; - my $info = ""; + # a device that showed up after the database got read is not in there + my $info = $udev_db ? $udev_db->{ basename($dev) } // '' : ''; my $data = {}; - eval { - run_command( - ['udevadm', 'info', '-p', $dev, '--query', 'all'], - outfunc => sub { - my ($line) = @_; - $info .= "$line\n"; - }, - ); - }; - warn $@ if $@; + if (!$info) { + eval { + run_command( + ['udevadm', 'info', '-p', $dev, '--query', 'all'], + outfunc => sub { + my ($line) = @_; + $info .= "$line\n"; + }, + ); + }; + warn $@ if $@; + } return if !$info; return if $info !~ m/^E: DEVTYPE=(disk|partition)$/m; @@ -594,6 +623,13 @@ sub get_disks { # through the same match, so perl would compile that regex again for every disk. $wanted = { map { $_ => 1 } @$disks }; } + # Reading the database of all block devices takes about as long as single queries for a 40th + # of the devices that lsblk lists, and a node with multipath maps or many volumes has far more + # of those than disks to list. So ask about single disks until that took as long as reading the + # database would, and about the first few in any case. + my ($udev_db, $udev_queries) = (undef, 0); + my $udev_single = scalar(keys %$lsblk_info) / 40; + $udev_single = 4 if $udev_single < 4; dir_glob_foreach( '/sys/block', @@ -615,7 +651,8 @@ sub get_disks { return if !$san_inventory && is_iscsi_device($dev); - my $data = get_udev_info("/sys/block/$dev") // return; + $udev_db //= get_udev_db() if ++$udev_queries > $udev_single; + my $data = get_udev_info("/sys/block/$dev", $udev_db) // return; my $devpath = $data->{devpath}; my $sysdir = "/sys/block/$dev"; diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm index 1ad1cf66..305bab8e 100644 --- a/src/test/disklist_test.pm +++ b/src/test/disklist_test.pm @@ -19,6 +19,7 @@ my $testcount = 0; # testcount for TAP::Harness my $diskmanage_module; # mockmodule for PVE::Diskmanage my $multipath_module; # mockmodule for PVE::Multipath my $print = 0; +my $udev_single_queries = 0; # udevadm calls for one device my $multipath_aliases = { '/dev/mapper/mpatha' => '/dev/dm-0', @@ -33,7 +34,18 @@ sub mocked_run_command { my $outputlines = []; if (my $ref = ref($cmd)) { - if ($cmd->[0] =~ m/udevadm/i) { + if ($cmd->[0] =~ m/udevadm/i && grep { $_ eq '--export-db' } @$cmd) { + # simulate the database of all devices, an empty line ends an entry + for my $file (sort glob("disk_tests/$testcasedir/*_udevadm")) { + my ($name) = $file =~ m|/([^/]+)$|; + my @lines = split(/\n/, read_test_file($name)); + # the database is keyed by the device path, which older fixtures lack + unshift @lines, 'P: /devices/virtual/block/' . ($name =~ s/_udevadm$//r) + if !grep { m/^P: / } @lines; + push @$outputlines, @lines, ''; + } + } elsif ($cmd->[0] =~ m/udevadm/i) { + $udev_single_queries++; # simulate udevadm output my $dev = $cmd->[3]; $dev =~ s|/sys/block/||; @@ -279,6 +291,58 @@ sub test_disk_list { is_deeply($status, $expected_status, 'nvme path status should be the same'); } + if ($testcasedir eq 'multipath') { + my $db = PVE::Diskmanage::get_udev_db(); + $testcount++; + is_deeply( + [sort keys %$db], + [qw(nvme8n1 nvme9n1 sda sdb sdc sdd)], + 'udev database has one entry per device', + ); + $udev_single_queries = 0; + my $info = PVE::Diskmanage::get_udev_info('/sys/block/sdb', $db); + $testcount++; + is($info->{serial}, 'SSERb', 'disk found in the udev database'); + $testcount++; + is($udev_single_queries, 0, 'without a query of its own'); + PVE::Diskmanage::get_udev_info('/sys/block/sdb', {}); + $testcount++; + is( + $udev_single_queries, + 1, + 'a disk that is not in the database is asked for on its own', + ); + + $udev_single_queries = 0; + my $listed = PVE::Diskmanage::get_disks(); + $testcount++; + is(scalar(keys %$listed), 6, 'all disks listed'); + $testcount++; + is( + $udev_single_queries, + 4, + 'the database replaces the queries after the first disks', + ); + + # six disks: 240 block devices are just enough to ask about each of them on its own + for my $count (240, 239) { + $diskmanage_module->mock( + get_lsblk_info => sub { + return { map { ("/dev/dm-$_" => {}) } 1 .. $count }; + }, + ); + $udev_single_queries = 0; + PVE::Diskmanage::get_disks(); + $diskmanage_module->unmock('get_lsblk_info'); + $testcount++; + is( + $udev_single_queries, + $count == 240 ? 6 : 5, + "single queries for six disks among $count block devices", + ); + } + } + done_testing($testcount); }; }