From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH storage 04/17] api: scan: add NVMe over Fabrics discovery
Date: Fri, 13 Nov 2026 04:04:32 +0100 [thread overview]
Message-ID: <20261113030541.1591528-5-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261113030541.1591528-1-t.lamprecht@proxmox.com>
Setting up an NVMe over Fabrics storage needs the NQN of the
subsystem and its portals. Add /nodes/{node}/scan/nvmeof, which asks
the discovery service at a portal for the NVMe/TCP subsystems it
offers to the node and lists them with their portals.
Without a port, it tries the discovery port 8009 and then the I/O
port 4420, where Linux targets answer discovery too. The scan needs
Datastore.Allocate on /storage, like the iSCSI scan.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/API2/Storage/Scan.pm | 49 +++++++++++++++++++++++
src/PVE/NVMeOF.pm | 62 ++++++++++++++++++++++++++++++
src/test/run_nvmeof_taint_tests.pl | 5 +++
src/test/run_nvmeof_tests.pl | 54 ++++++++++++++++++++++++++
4 files changed, 170 insertions(+)
diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm
index 96e1bd34..a9d05f79 100644
--- a/src/PVE/API2/Storage/Scan.pm
+++ b/src/PVE/API2/Storage/Scan.pm
@@ -6,6 +6,7 @@ use warnings;
# NOTE: This API endpoints are mounted by pve-manager's API2::Node module and pvesm CLI
use PVE::JSONSchema qw(get_standard_option);
+use PVE::NVMeOF;
use PVE::RESTHandler;
use PVE::SafeSyslog;
@@ -49,6 +50,7 @@ __PACKAGE__->register_method({
{ method => 'iscsi' },
{ method => 'lvm' },
{ method => 'nfs' },
+ { method => 'nvmeof' },
{ method => 'pbs' },
{ method => 'san-luns' },
{ method => 'zfs' },
@@ -299,6 +301,53 @@ __PACKAGE__->register_method({
},
});
+__PACKAGE__->register_method({
+ name => 'nvmeofscan',
+ path => 'nvmeof',
+ method => 'GET',
+ description =>
+ 'List the NVMe over TCP subsystems offered to this node by a discovery controller.',
+ protected => 1,
+ proxyto => 'node',
+ permissions => { check => ['perm', '/storage', ['Datastore.Allocate']] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ portal => {
+ description => 'IPv4 or bracketed IPv6 discovery address with an optional port.'
+ . ' Without a port, try 8009 then 4420.',
+ type => 'string',
+ format => 'pve-storage-nvme-portal',
+ },
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => 'object',
+ additionalProperties => 0,
+ properties => {
+ subsysnqn => { type => 'string', description => 'The subsystem NQN.' },
+ portals => {
+ type => 'string',
+ description => 'The discovered TCP portals, comma separated.',
+ },
+ },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+ my $res = PVE::NVMeOF::discover($param->{portal});
+ return [
+ map { {
+ subsysnqn => $_,
+ portals => join(',', map { PVE::NVMeOF::format_path($_) } $res->{$_}->@*),
+ } } sort keys %$res
+ ];
+ },
+});
+
__PACKAGE__->register_method({
name => 'lvmscan',
path => 'lvm',
diff --git a/src/PVE/NVMeOF.pm b/src/PVE/NVMeOF.pm
index 7a3140cb..d1dfe15b 100644
--- a/src/PVE/NVMeOF.pm
+++ b/src/PVE/NVMeOF.pm
@@ -5,10 +5,12 @@ use v5.36;
use Digest::SHA qw(sha1_hex);
use Fcntl qw(:flock F_GETFD F_SETFD FD_CLOEXEC);
use File::Path qw(make_path);
+use JSON qw(decode_json);
use Socket qw(AF_INET AF_INET6 inet_ntop inet_pton);
use PVE::Diskmanage;
use PVE::JSONSchema;
+use PVE::Network;
use PVE::Tools qw(dir_glob_foreach file_read_firstline file_set_contents run_command);
our $NVME = '/usr/sbin/nvme';
@@ -246,4 +248,64 @@ sub connect_path($scfg, $path) {
});
}
+sub parse_discovery_log($raw) {
+ my $log = decode_json($raw);
+ die "invalid NVMe discovery log\n"
+ if ref($log) ne 'HASH' || ref($log->{records}) ne 'ARRAY';
+ my (%seen, %res);
+ for my $record ($log->{records}->@*) {
+ next
+ if ($record->{subtype} // '') ne 'nvme subsystem' || ($record->{trtype} // '') ne 'tcp';
+ my $nqn = $record->{subnqn} // next;
+ next
+ if $nqn !~ /^$NQN_PATTERN$/
+ || length($nqn) > 223
+ || $nqn eq 'nqn.2014-08.org.nvmexpress.discovery';
+ my $addr = $record->{traddr} // '';
+ my $port = $record->{trsvcid} // '';
+ my $path = parse_portal(($addr =~ /:/ ? "[$addr]" : $addr) . ":$port") // next;
+ next if $seen{$nqn}->{ format_path($path) }++;
+ push $res{$nqn}->@*, $path;
+ }
+ return \%res;
+}
+
+sub discover($portal) {
+ my $path = parse_portal($portal) // die "invalid portal '$portal'\n";
+ my @ports = $portal =~ /:\d+$/ ? ($path->{trsvcid}) : (8009, 4420);
+ my $error;
+ for my $port (@ports) {
+ my $result = eval {
+ die "no answer from portal\n" if !PVE::Network::tcp_ping($path->{traddr}, $port, 2);
+ with_lock(sub {
+ assert_support();
+ my $host = host_identity();
+ my $out = '';
+ run_command(
+ [
+ $NVME,
+ 'discover',
+ '--transport=tcp',
+ '--traddr',
+ $path->{traddr},
+ '--trsvcid',
+ $port,
+ '--hostnqn',
+ $host->{nqn},
+ '--hostid',
+ $host->{id},
+ '--output-format=json',
+ ],
+ timeout => 30,
+ outfunc => sub($line) { $out .= "$line\n" },
+ );
+ return parse_discovery_log($out);
+ });
+ };
+ $error = $@;
+ return $result if !$error;
+ }
+ die "discovery at portal '$portal' failed - $error";
+}
+
1;
diff --git a/src/test/run_nvmeof_taint_tests.pl b/src/test/run_nvmeof_taint_tests.pl
index 3c19f8cc..d136c4f1 100755
--- a/src/test/run_nvmeof_taint_tests.pl
+++ b/src/test/run_nvmeof_taint_tests.pl
@@ -38,6 +38,7 @@ $nvme->mock(
'all connect operands are validated and untainted',
);
PVE::Tools::run_command(['/usr/bin/true', @$cmd], timeout => 5);
+ $opts{outfunc}->('{"records":[]}') if $opts{outfunc};
},
);
my $path = PVE::NVMeOF::parse_portal($portal);
@@ -53,6 +54,10 @@ for my $bad ('nqn.bad,nqn=other', "nqn.test\n", 'nqn.2014-08.org.nvmexpress.disc
eval { PVE::NVMeOF::connect_path({ subsysnqn => $bad }, $path) };
like($@, qr/invalid subsystem/, 'invalid connect subsystem refused');
}
+my $net = Test::MockModule->new('PVE::Network');
+$net->mock(tcp_ping => sub { 1 });
+eval { PVE::NVMeOF::discover($portal) };
+is($@, '', 'discovery can execute with file-backed host identity in taint mode');
file_set_contents($PVE::NVMeOF::HOSTID_FILE, "00000000-0000-0000-0000-000000000000\n");
eval { PVE::NVMeOF::host_identity() };
like($@, qr/nonzero UUID/, 'zero host ID refused');
diff --git a/src/test/run_nvmeof_tests.pl b/src/test/run_nvmeof_tests.pl
index 5ff96833..ead8d03c 100755
--- a/src/test/run_nvmeof_tests.pl
+++ b/src/test/run_nvmeof_tests.pl
@@ -9,6 +9,8 @@ use Test::More;
use Test::MockModule;
use PVE::NVMeOF;
use PVE::Storage;
+use PVE::API2::Storage::Scan;
+use JSON qw(encode_json);
use PVE::Tools qw(file_set_contents);
my $dir = tempdir('nvmeof-test-XXXX', DIR => '.', CLEANUP => 1);
@@ -240,4 +242,56 @@ subtest 'local disk setup rejects fabric namespaces by transport' => sub {
}
};
+subtest 'TCP discovery, fallback and API result' => sub {
+ my $record = {
+ subtype => 'nvme subsystem',
+ trtype => 'tcp',
+ subnqn => 'nqn.test',
+ traddr => '192.0.2.1',
+ trsvcid => '4420',
+ };
+ my $raw = encode_json({
+ records =>
+ [$record, $record, { %$record, trtype => 'rdma' }, { %$record, trsvcid => 70000 }],
+ });
+ is_deeply(
+ PVE::NVMeOF::parse_discovery_log($raw),
+ { 'nqn.test' => [{ traddr => '192.0.2.1', trsvcid => 4420 }] },
+ 'TCP only, unique valid portals',
+ );
+ for my $bad ('garbage', '{}', '[]') {
+ eval { PVE::NVMeOF::parse_discovery_log($bad) };
+ ok($@, 'malformed discovery is not an empty success');
+ }
+ my $nvme = Test::MockModule->new('PVE::NVMeOF');
+ my $net = Test::MockModule->new('PVE::Network');
+ my (@ports, @commands);
+ our $locked;
+ $nvme->mock(with_lock => sub { local $locked = 1; $_[0]->() });
+ $nvme->mock(assert_support => sub { });
+ $nvme->mock(host_identity => sub { { nqn => 'nqn.host', id => 'id' } });
+ $net->mock(tcp_ping => sub { push @ports, $_[1]; $_[1] != 8009 });
+ $nvme->mock(
+ run_command => sub($cmd, %opts) {
+ ok($locked, 'discovery holds the NVMe lock');
+ is($opts{timeout}, 30, 'discovery command timeout');
+ push @commands, $cmd;
+ $opts{outfunc}->($raw);
+ },
+ );
+ my $rows = PVE::API2::Storage::Scan->nvmeofscan({ node => 'n1', portal => '192.0.2.1' });
+ is_deeply(\@ports, [8009, 4420], 'unspecified port tries discovery then I/O port');
+ is_deeply($rows, [{ subsysnqn => 'nqn.test', portals => '192.0.2.1:4420' }], 'API shape');
+ eval {
+ PVE::JSONSchema::validate(
+ $rows,
+ PVE::API2::Storage::Scan->map_method_by_name('nvmeofscan')->{returns},
+ );
+ };
+ is($@, '', 'API schema accepts result');
+ @ports = ();
+ PVE::NVMeOF::discover('[2001:db8::1]:4421');
+ is_deeply(\@ports, [4421], 'explicit port does not fall back');
+};
+
done_testing();
next prev parent reply other threads:[~2026-11-13 3:25 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-13 3:04 [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 01/17] diskmanage: skip SMART queries for NVMe fabric namespaces Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 02/17] nvmeof: add a storage for NVMe over Fabrics subsystems Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 03/17] lvm: allow a volume group on an NVMe over Fabrics namespace Thomas Lamprecht
2026-11-13 3:04 ` Thomas Lamprecht [this message]
2026-11-13 3:04 ` [PATCH storage 05/17] pvesm: add a scan command for NVMe over Fabrics Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 06/17] api: scan: san-luns: attribute NVMe namespaces to storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 07/17] api: add node-level NVMe over Fabrics status and controller removal Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 08/17] multipath: report the NVMe host NQN of each node Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 09/17] multipath: report NVMe over Fabrics namespaces in the cluster status Thomas Lamprecht
2026-11-13 3:04 ` [PATCH docs 10/17] storage: document NVMe over Fabrics storages Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 11/17] api: nodes: mount the NVMe over Fabrics endpoints Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 12/17] ui: storage: add NVMe over Fabrics storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 13/17] ui: lvm: offer NVMe over Fabrics storages as base storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 14/17] ui: dc: san: add a guided setup for shared LVM on NVMe over Fabrics Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 15/17] ui: node: san: list NVMe over Fabrics controllers Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 16/17] ui: dc: san: show the NVMe host NQN of each node Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 17/17] ui: dc: san: show that a namespace uses native multipath Thomas Lamprecht
2026-11-18 20:44 ` superseded: [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261113030541.1591528-5-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox