From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 6300F1FF0B7 for ; Fri, 13 Nov 2026 04:25:28 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 53F812196A; Fri, 13 Nov 2026 04:21:59 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH storage 04/17] api: scan: add NVMe over Fabrics discovery Date: Fri, 13 Nov 2026 04:04:32 +0100 Message-ID: <20261113030541.1591528-5-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113030541.1591528-1-t.lamprecht@proxmox.com> References: <20261113030541.1591528-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794539148164 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.917 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: RIMZAVAUZ5OB3RBAC3DELYLZIE4XEBQX X-Message-ID-Hash: RIMZAVAUZ5OB3RBAC3DELYLZIE4XEBQX X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Setting up an NVMe over Fabrics storage needs the NQN of the subsystem and its portals. Add /nodes/{node}/scan/nvmeof, which asks the discovery service at a portal for the NVMe/TCP subsystems it offers to the node and lists them with their portals. Without a port, it tries the discovery port 8009 and then the I/O port 4420, where Linux targets answer discovery too. The scan needs Datastore.Allocate on /storage, like the iSCSI scan. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Storage/Scan.pm | 49 +++++++++++++++++++++++ src/PVE/NVMeOF.pm | 62 ++++++++++++++++++++++++++++++ src/test/run_nvmeof_taint_tests.pl | 5 +++ src/test/run_nvmeof_tests.pl | 54 ++++++++++++++++++++++++++ 4 files changed, 170 insertions(+) diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm index 96e1bd34..a9d05f79 100644 --- a/src/PVE/API2/Storage/Scan.pm +++ b/src/PVE/API2/Storage/Scan.pm @@ -6,6 +6,7 @@ use warnings; # NOTE: This API endpoints are mounted by pve-manager's API2::Node module and pvesm CLI use PVE::JSONSchema qw(get_standard_option); +use PVE::NVMeOF; use PVE::RESTHandler; use PVE::SafeSyslog; @@ -49,6 +50,7 @@ __PACKAGE__->register_method({ { method => 'iscsi' }, { method => 'lvm' }, { method => 'nfs' }, + { method => 'nvmeof' }, { method => 'pbs' }, { method => 'san-luns' }, { method => 'zfs' }, @@ -299,6 +301,53 @@ __PACKAGE__->register_method({ }, }); +__PACKAGE__->register_method({ + name => 'nvmeofscan', + path => 'nvmeof', + method => 'GET', + description => + 'List the NVMe over TCP subsystems offered to this node by a discovery controller.', + protected => 1, + proxyto => 'node', + permissions => { check => ['perm', '/storage', ['Datastore.Allocate']] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + portal => { + description => 'IPv4 or bracketed IPv6 discovery address with an optional port.' + . ' Without a port, try 8009 then 4420.', + type => 'string', + format => 'pve-storage-nvme-portal', + }, + }, + }, + returns => { + type => 'array', + items => { + type => 'object', + additionalProperties => 0, + properties => { + subsysnqn => { type => 'string', description => 'The subsystem NQN.' }, + portals => { + type => 'string', + description => 'The discovered TCP portals, comma separated.', + }, + }, + }, + }, + code => sub { + my ($param) = @_; + my $res = PVE::NVMeOF::discover($param->{portal}); + return [ + map { { + subsysnqn => $_, + portals => join(',', map { PVE::NVMeOF::format_path($_) } $res->{$_}->@*), + } } sort keys %$res + ]; + }, +}); + __PACKAGE__->register_method({ name => 'lvmscan', path => 'lvm', diff --git a/src/PVE/NVMeOF.pm b/src/PVE/NVMeOF.pm index 7a3140cb..d1dfe15b 100644 --- a/src/PVE/NVMeOF.pm +++ b/src/PVE/NVMeOF.pm @@ -5,10 +5,12 @@ use v5.36; use Digest::SHA qw(sha1_hex); use Fcntl qw(:flock F_GETFD F_SETFD FD_CLOEXEC); use File::Path qw(make_path); +use JSON qw(decode_json); use Socket qw(AF_INET AF_INET6 inet_ntop inet_pton); use PVE::Diskmanage; use PVE::JSONSchema; +use PVE::Network; use PVE::Tools qw(dir_glob_foreach file_read_firstline file_set_contents run_command); our $NVME = '/usr/sbin/nvme'; @@ -246,4 +248,64 @@ sub connect_path($scfg, $path) { }); } +sub parse_discovery_log($raw) { + my $log = decode_json($raw); + die "invalid NVMe discovery log\n" + if ref($log) ne 'HASH' || ref($log->{records}) ne 'ARRAY'; + my (%seen, %res); + for my $record ($log->{records}->@*) { + next + if ($record->{subtype} // '') ne 'nvme subsystem' || ($record->{trtype} // '') ne 'tcp'; + my $nqn = $record->{subnqn} // next; + next + if $nqn !~ /^$NQN_PATTERN$/ + || length($nqn) > 223 + || $nqn eq 'nqn.2014-08.org.nvmexpress.discovery'; + my $addr = $record->{traddr} // ''; + my $port = $record->{trsvcid} // ''; + my $path = parse_portal(($addr =~ /:/ ? "[$addr]" : $addr) . ":$port") // next; + next if $seen{$nqn}->{ format_path($path) }++; + push $res{$nqn}->@*, $path; + } + return \%res; +} + +sub discover($portal) { + my $path = parse_portal($portal) // die "invalid portal '$portal'\n"; + my @ports = $portal =~ /:\d+$/ ? ($path->{trsvcid}) : (8009, 4420); + my $error; + for my $port (@ports) { + my $result = eval { + die "no answer from portal\n" if !PVE::Network::tcp_ping($path->{traddr}, $port, 2); + with_lock(sub { + assert_support(); + my $host = host_identity(); + my $out = ''; + run_command( + [ + $NVME, + 'discover', + '--transport=tcp', + '--traddr', + $path->{traddr}, + '--trsvcid', + $port, + '--hostnqn', + $host->{nqn}, + '--hostid', + $host->{id}, + '--output-format=json', + ], + timeout => 30, + outfunc => sub($line) { $out .= "$line\n" }, + ); + return parse_discovery_log($out); + }); + }; + $error = $@; + return $result if !$error; + } + die "discovery at portal '$portal' failed - $error"; +} + 1; diff --git a/src/test/run_nvmeof_taint_tests.pl b/src/test/run_nvmeof_taint_tests.pl index 3c19f8cc..d136c4f1 100755 --- a/src/test/run_nvmeof_taint_tests.pl +++ b/src/test/run_nvmeof_taint_tests.pl @@ -38,6 +38,7 @@ $nvme->mock( 'all connect operands are validated and untainted', ); PVE::Tools::run_command(['/usr/bin/true', @$cmd], timeout => 5); + $opts{outfunc}->('{"records":[]}') if $opts{outfunc}; }, ); my $path = PVE::NVMeOF::parse_portal($portal); @@ -53,6 +54,10 @@ for my $bad ('nqn.bad,nqn=other', "nqn.test\n", 'nqn.2014-08.org.nvmexpress.disc eval { PVE::NVMeOF::connect_path({ subsysnqn => $bad }, $path) }; like($@, qr/invalid subsystem/, 'invalid connect subsystem refused'); } +my $net = Test::MockModule->new('PVE::Network'); +$net->mock(tcp_ping => sub { 1 }); +eval { PVE::NVMeOF::discover($portal) }; +is($@, '', 'discovery can execute with file-backed host identity in taint mode'); file_set_contents($PVE::NVMeOF::HOSTID_FILE, "00000000-0000-0000-0000-000000000000\n"); eval { PVE::NVMeOF::host_identity() }; like($@, qr/nonzero UUID/, 'zero host ID refused'); diff --git a/src/test/run_nvmeof_tests.pl b/src/test/run_nvmeof_tests.pl index 5ff96833..ead8d03c 100755 --- a/src/test/run_nvmeof_tests.pl +++ b/src/test/run_nvmeof_tests.pl @@ -9,6 +9,8 @@ use Test::More; use Test::MockModule; use PVE::NVMeOF; use PVE::Storage; +use PVE::API2::Storage::Scan; +use JSON qw(encode_json); use PVE::Tools qw(file_set_contents); my $dir = tempdir('nvmeof-test-XXXX', DIR => '.', CLEANUP => 1); @@ -240,4 +242,56 @@ subtest 'local disk setup rejects fabric namespaces by transport' => sub { } }; +subtest 'TCP discovery, fallback and API result' => sub { + my $record = { + subtype => 'nvme subsystem', + trtype => 'tcp', + subnqn => 'nqn.test', + traddr => '192.0.2.1', + trsvcid => '4420', + }; + my $raw = encode_json({ + records => + [$record, $record, { %$record, trtype => 'rdma' }, { %$record, trsvcid => 70000 }], + }); + is_deeply( + PVE::NVMeOF::parse_discovery_log($raw), + { 'nqn.test' => [{ traddr => '192.0.2.1', trsvcid => 4420 }] }, + 'TCP only, unique valid portals', + ); + for my $bad ('garbage', '{}', '[]') { + eval { PVE::NVMeOF::parse_discovery_log($bad) }; + ok($@, 'malformed discovery is not an empty success'); + } + my $nvme = Test::MockModule->new('PVE::NVMeOF'); + my $net = Test::MockModule->new('PVE::Network'); + my (@ports, @commands); + our $locked; + $nvme->mock(with_lock => sub { local $locked = 1; $_[0]->() }); + $nvme->mock(assert_support => sub { }); + $nvme->mock(host_identity => sub { { nqn => 'nqn.host', id => 'id' } }); + $net->mock(tcp_ping => sub { push @ports, $_[1]; $_[1] != 8009 }); + $nvme->mock( + run_command => sub($cmd, %opts) { + ok($locked, 'discovery holds the NVMe lock'); + is($opts{timeout}, 30, 'discovery command timeout'); + push @commands, $cmd; + $opts{outfunc}->($raw); + }, + ); + my $rows = PVE::API2::Storage::Scan->nvmeofscan({ node => 'n1', portal => '192.0.2.1' }); + is_deeply(\@ports, [8009, 4420], 'unspecified port tries discovery then I/O port'); + is_deeply($rows, [{ subsysnqn => 'nqn.test', portals => '192.0.2.1:4420' }], 'API shape'); + eval { + PVE::JSONSchema::validate( + $rows, + PVE::API2::Storage::Scan->map_method_by_name('nvmeofscan')->{returns}, + ); + }; + is($@, '', 'API schema accepts result'); + @ports = (); + PVE::NVMeOF::discover('[2001:db8::1]:4421'); + is_deeply(\@ports, [4421], 'explicit port does not fall back'); +}; + done_testing();