From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH storage 04/17] api: scan: add NVMe over Fabrics discovery
Date: Fri, 13 Nov 2026 04:04:32 +0100 [thread overview]
Message-ID: <20261113030541.1591528-5-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261113030541.1591528-1-t.lamprecht@proxmox.com>
Setting up an NVMe over Fabrics storage needs the NQN of the
subsystem and its portals. Add /nodes/{node}/scan/nvmeof, which asks
the discovery service at a portal for the NVMe/TCP subsystems it
offers to the node and lists them with their portals.
Without a port, it tries the discovery port 8009 and then the I/O
port 4420, where Linux targets answer discovery too. The scan needs
Datastore.Allocate on /storage, like the iSCSI scan.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/API2/Storage/Scan.pm | 49 +++++++++++++++++++++++
src/PVE/NVMeOF.pm | 62 ++++++++++++++++++++++++++++++
src/test/run_nvmeof_taint_tests.pl | 5 +++
src/test/run_nvmeof_tests.pl | 54 ++++++++++++++++++++++++++
4 files changed, 170 insertions(+)
diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm
index 96e1bd34..a9d05f79 100644
--- a/src/PVE/API2/Storage/Scan.pm
+++ b/src/PVE/API2/Storage/Scan.pm
@@ -6,6 +6,7 @@ use warnings;
# NOTE: This API endpoints are mounted by pve-manager's API2::Node module and pvesm CLI
use PVE::JSONSchema qw(get_standard_option);
+use PVE::NVMeOF;
use PVE::RESTHandler;
use PVE::SafeSyslog;
@@ -49,6 +50,7 @@ __PACKAGE__->register_method({
{ method => 'iscsi' },
{ method => 'lvm' },
{ method => 'nfs' },
+ { method => 'nvmeof' },
{ method => 'pbs' },
{ method => 'san-luns' },
{ method => 'zfs' },
@@ -299,6 +301,53 @@ __PACKAGE__->register_method({
},
});
+__PACKAGE__->register_method({
+ name => 'nvmeofscan',
+ path => 'nvmeof',
+ method => 'GET',
+ description =>
+ 'List the NVMe over TCP subsystems offered to this node by a discovery controller.',
+ protected => 1,
+ proxyto => 'node',
+ permissions => { check => ['perm', '/storage', ['Datastore.Allocate']] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ portal => {
+ description => 'IPv4 or bracketed IPv6 discovery address with an optional port.'
+ . ' Without a port, try 8009 then 4420.',
+ type => 'string',
+ format => 'pve-storage-nvme-portal',
+ },
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => 'object',
+ additionalProperties => 0,
+ properties => {
+ subsysnqn => { type => 'string', description => 'The subsystem NQN.' },
+ portals => {
+ type => 'string',
+ description => 'The discovered TCP portals, comma separated.',
+ },
+ },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+ my $res = PVE::NVMeOF::discover($param->{portal});
+ return [
+ map { {
+ subsysnqn => $_,
+ portals => join(',', map { PVE::NVMeOF::format_path($_) } $res->{$_}->@*),
+ } } sort keys %$res
+ ];
+ },
+});
+
__PACKAGE__->register_method({
name => 'lvmscan',
path => 'lvm',
diff --git a/src/PVE/NVMeOF.pm b/src/PVE/NVMeOF.pm
index 7a3140cb..d1dfe15b 100644
--- a/src/PVE/NVMeOF.pm
+++ b/src/PVE/NVMeOF.pm
@@ -5,10 +5,12 @@ use v5.36;
use Digest::SHA qw(sha1_hex);
use Fcntl qw(:flock F_GETFD F_SETFD FD_CLOEXEC);
use File::Path qw(make_path);
+use JSON qw(decode_json);
use Socket qw(AF_INET AF_INET6 inet_ntop inet_pton);
use PVE::Diskmanage;
use PVE::JSONSchema;
+use PVE::Network;
use PVE::Tools qw(dir_glob_foreach file_read_firstline file_set_contents run_command);
our $NVME = '/usr/sbin/nvme';
@@ -246,4 +248,64 @@ sub connect_path($scfg, $path) {
});
}
+sub parse_discovery_log($raw) {
+ my $log = decode_json($raw);
+ die "invalid NVMe discovery log\n"
+ if ref($log) ne 'HASH' || ref($log->{records}) ne 'ARRAY';
+ my (%seen, %res);
+ for my $record ($log->{records}->@*) {
+ next
+ if ($record->{subtype} // '') ne 'nvme subsystem' || ($record->{trtype} // '') ne 'tcp';
+ my $nqn = $record->{subnqn} // next;
+ next
+ if $nqn !~ /^$NQN_PATTERN$/
+ || length($nqn) > 223
+ || $nqn eq 'nqn.2014-08.org.nvmexpress.discovery';
+ my $addr = $record->{traddr} // '';
+ my $port = $record->{trsvcid} // '';
+ my $path = parse_portal(($addr =~ /:/ ? "[$addr]" : $addr) . ":$port") // next;
+ next if $seen{$nqn}->{ format_path($path) }++;
+ push $res{$nqn}->@*, $path;
+ }
+ return \%res;
+}
+
+sub discover($portal) {
+ my $path = parse_portal($portal) // die "invalid portal '$portal'\n";
+ my @ports = $portal =~ /:\d+$/ ? ($path->{trsvcid}) : (8009, 4420);
+ my $error;
+ for my $port (@ports) {
+ my $result = eval {
+ die "no answer from portal\n" if !PVE::Network::tcp_ping($path->{traddr}, $port, 2);
+ with_lock(sub {
+ assert_support();
+ my $host = host_identity();
+ my $out = '';
+ run_command(
+ [
+ $NVME,
+ 'discover',
+ '--transport=tcp',
+ '--traddr',
+ $path->{traddr},
+ '--trsvcid',
+ $port,
+ '--hostnqn',
+ $host->{nqn},
+ '--hostid',
+ $host->{id},
+ '--output-format=json',
+ ],
+ timeout => 30,
+ outfunc => sub($line) { $out .= "$line\n" },
+ );
+ return parse_discovery_log($out);
+ });
+ };
+ $error = $@;
+ return $result if !$error;
+ }
+ die "discovery at portal '$portal' failed - $error";
+}
+
1;
diff --git a/src/test/run_nvmeof_taint_tests.pl b/src/test/run_nvmeof_taint_tests.pl
index 3c19f8cc..d136c4f1 100755
--- a/src/test/run_nvmeof_taint_tests.pl
+++ b/src/test/run_nvmeof_taint_tests.pl
@@ -38,6 +38,7 @@ $nvme->mock(
'all connect operands are validated and untainted',
);
PVE::Tools::run_command(['/usr/bin/true', @$cmd], timeout => 5);
+ $opts{outfunc}->('{"records":[]}') if $opts{outfunc};
},
);
my $path = PVE::NVMeOF::parse_portal($portal);
@@ -53,6 +54,10 @@ for my $bad ('nqn.bad,nqn=other', "nqn.test\n", 'nqn.2014-08.org.nvmexpress.disc
eval { PVE::NVMeOF::connect_path({ subsysnqn => $bad }, $path) };
like($@, qr/invalid subsystem/, 'invalid connect subsystem refused');
}
+my $net = Test::MockModule->new('PVE::Network');
+$net->mock(tcp_ping => sub { 1 });
+eval { PVE::NVMeOF::discover($portal) };
+is($@, '', 'discovery can execute with file-backed host identity in taint mode');
file_set_contents($PVE::NVMeOF::HOSTID_FILE, "00000000-0000-0000-0000-000000000000\n");
eval { PVE::NVMeOF::host_identity() };
like($@, qr/nonzero UUID/, 'zero host ID refused');
diff --git a/src/test/run_nvmeof_tests.pl b/src/test/run_nvmeof_tests.pl
index 5ff96833..ead8d03c 100755
--- a/src/test/run_nvmeof_tests.pl
+++ b/src/test/run_nvmeof_tests.pl
@@ -9,6 +9,8 @@ use Test::More;
use Test::MockModule;
use PVE::NVMeOF;
use PVE::Storage;
+use PVE::API2::Storage::Scan;
+use JSON qw(encode_json);
use PVE::Tools qw(file_set_contents);
my $dir = tempdir('nvmeof-test-XXXX', DIR => '.', CLEANUP => 1);
@@ -240,4 +242,56 @@ subtest 'local disk setup rejects fabric namespaces by transport' => sub {
}
};
+subtest 'TCP discovery, fallback and API result' => sub {
+ my $record = {
+ subtype => 'nvme subsystem',
+ trtype => 'tcp',
+ subnqn => 'nqn.test',
+ traddr => '192.0.2.1',
+ trsvcid => '4420',
+ };
+ my $raw = encode_json({
+ records =>
+ [$record, $record, { %$record, trtype => 'rdma' }, { %$record, trsvcid => 70000 }],
+ });
+ is_deeply(
+ PVE::NVMeOF::parse_discovery_log($raw),
+ { 'nqn.test' => [{ traddr => '192.0.2.1', trsvcid => 4420 }] },
+ 'TCP only, unique valid portals',
+ );
+ for my $bad ('garbage', '{}', '[]') {
+ eval { PVE::NVMeOF::parse_discovery_log($bad) };
+ ok($@, 'malformed discovery is not an empty success');
+ }
+ my $nvme = Test::MockModule->new('PVE::NVMeOF');
+ my $net = Test::MockModule->new('PVE::Network');
+ my (@ports, @commands);
+ our $locked;
+ $nvme->mock(with_lock => sub { local $locked = 1; $_[0]->() });
+ $nvme->mock(assert_support => sub { });
+ $nvme->mock(host_identity => sub { { nqn => 'nqn.host', id => 'id' } });
+ $net->mock(tcp_ping => sub { push @ports, $_[1]; $_[1] != 8009 });
+ $nvme->mock(
+ run_command => sub($cmd, %opts) {
+ ok($locked, 'discovery holds the NVMe lock');
+ is($opts{timeout}, 30, 'discovery command timeout');
+ push @commands, $cmd;
+ $opts{outfunc}->($raw);
+ },
+ );
+ my $rows = PVE::API2::Storage::Scan->nvmeofscan({ node => 'n1', portal => '192.0.2.1' });
+ is_deeply(\@ports, [8009, 4420], 'unspecified port tries discovery then I/O port');
+ is_deeply($rows, [{ subsysnqn => 'nqn.test', portals => '192.0.2.1:4420' }], 'API shape');
+ eval {
+ PVE::JSONSchema::validate(
+ $rows,
+ PVE::API2::Storage::Scan->map_method_by_name('nvmeofscan')->{returns},
+ );
+ };
+ is($@, '', 'API schema accepts result');
+ @ports = ();
+ PVE::NVMeOF::discover('[2001:db8::1]:4421');
+ is_deeply(\@ports, [4421], 'explicit port does not fall back');
+};
+
done_testing();
next prev parent reply other threads:[~2026-11-13 3:25 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-13 3:04 [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 01/17] diskmanage: skip SMART queries for NVMe fabric namespaces Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 02/17] nvmeof: add a storage for NVMe over Fabrics subsystems Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 03/17] lvm: allow a volume group on an NVMe over Fabrics namespace Thomas Lamprecht
2026-11-13 3:04 ` Thomas Lamprecht [this message]
2026-11-13 3:04 ` [PATCH storage 05/17] pvesm: add a scan command for NVMe over Fabrics Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 06/17] api: scan: san-luns: attribute NVMe namespaces to storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 07/17] api: add node-level NVMe over Fabrics status and controller removal Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 08/17] multipath: report the NVMe host NQN of each node Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 09/17] multipath: report NVMe over Fabrics namespaces in the cluster status Thomas Lamprecht
2026-11-13 3:04 ` [PATCH docs 10/17] storage: document NVMe over Fabrics storages Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 11/17] api: nodes: mount the NVMe over Fabrics endpoints Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 12/17] ui: storage: add NVMe over Fabrics storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 13/17] ui: lvm: offer NVMe over Fabrics storages as base storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 14/17] ui: dc: san: add a guided setup for shared LVM on NVMe over Fabrics Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 15/17] ui: node: san: list NVMe over Fabrics controllers Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 16/17] ui: dc: san: show the NVMe host NQN of each node Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 17/17] ui: dc: san: show that a namespace uses native multipath Thomas Lamprecht
2026-11-18 20:44 ` superseded: [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261113030541.1591528-5-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.