public inbox for pve-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH storage 04/17] api: scan: add NVMe over Fabrics discovery
Date: Fri, 13 Nov 2026 04:04:32 +0100	[thread overview]
Message-ID: <20261113030541.1591528-5-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261113030541.1591528-1-t.lamprecht@proxmox.com>

Setting up an NVMe over Fabrics storage needs the NQN of the
subsystem and its portals. Add /nodes/{node}/scan/nvmeof, which asks
the discovery service at a portal for the NVMe/TCP subsystems it
offers to the node and lists them with their portals.

Without a port, it tries the discovery port 8009 and then the I/O
port 4420, where Linux targets answer discovery too. The scan needs
Datastore.Allocate on /storage, like the iSCSI scan.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 src/PVE/API2/Storage/Scan.pm       | 49 +++++++++++++++++++++++
 src/PVE/NVMeOF.pm                  | 62 ++++++++++++++++++++++++++++++
 src/test/run_nvmeof_taint_tests.pl |  5 +++
 src/test/run_nvmeof_tests.pl       | 54 ++++++++++++++++++++++++++
 4 files changed, 170 insertions(+)

diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm
index 96e1bd34..a9d05f79 100644
--- a/src/PVE/API2/Storage/Scan.pm
+++ b/src/PVE/API2/Storage/Scan.pm
@@ -6,6 +6,7 @@ use warnings;
 # NOTE: This API endpoints are mounted by pve-manager's API2::Node module and pvesm CLI
 
 use PVE::JSONSchema qw(get_standard_option);
+use PVE::NVMeOF;
 use PVE::RESTHandler;
 use PVE::SafeSyslog;
 
@@ -49,6 +50,7 @@ __PACKAGE__->register_method({
             { method => 'iscsi' },
             { method => 'lvm' },
             { method => 'nfs' },
+            { method => 'nvmeof' },
             { method => 'pbs' },
             { method => 'san-luns' },
             { method => 'zfs' },
@@ -299,6 +301,53 @@ __PACKAGE__->register_method({
     },
 });
 
+__PACKAGE__->register_method({
+    name => 'nvmeofscan',
+    path => 'nvmeof',
+    method => 'GET',
+    description =>
+        'List the NVMe over TCP subsystems offered to this node by a discovery controller.',
+    protected => 1,
+    proxyto => 'node',
+    permissions => { check => ['perm', '/storage', ['Datastore.Allocate']] },
+    parameters => {
+        additionalProperties => 0,
+        properties => {
+            node => get_standard_option('pve-node'),
+            portal => {
+                description => 'IPv4 or bracketed IPv6 discovery address with an optional port.'
+                    . ' Without a port, try 8009 then 4420.',
+                type => 'string',
+                format => 'pve-storage-nvme-portal',
+            },
+        },
+    },
+    returns => {
+        type => 'array',
+        items => {
+            type => 'object',
+            additionalProperties => 0,
+            properties => {
+                subsysnqn => { type => 'string', description => 'The subsystem NQN.' },
+                portals => {
+                    type => 'string',
+                    description => 'The discovered TCP portals, comma separated.',
+                },
+            },
+        },
+    },
+    code => sub {
+        my ($param) = @_;
+        my $res = PVE::NVMeOF::discover($param->{portal});
+        return [
+            map { {
+                subsysnqn => $_,
+                portals => join(',', map { PVE::NVMeOF::format_path($_) } $res->{$_}->@*),
+            } } sort keys %$res
+        ];
+    },
+});
+
 __PACKAGE__->register_method({
     name => 'lvmscan',
     path => 'lvm',
diff --git a/src/PVE/NVMeOF.pm b/src/PVE/NVMeOF.pm
index 7a3140cb..d1dfe15b 100644
--- a/src/PVE/NVMeOF.pm
+++ b/src/PVE/NVMeOF.pm
@@ -5,10 +5,12 @@ use v5.36;
 use Digest::SHA qw(sha1_hex);
 use Fcntl qw(:flock F_GETFD F_SETFD FD_CLOEXEC);
 use File::Path qw(make_path);
+use JSON qw(decode_json);
 use Socket qw(AF_INET AF_INET6 inet_ntop inet_pton);
 
 use PVE::Diskmanage;
 use PVE::JSONSchema;
+use PVE::Network;
 use PVE::Tools qw(dir_glob_foreach file_read_firstline file_set_contents run_command);
 
 our $NVME = '/usr/sbin/nvme';
@@ -246,4 +248,64 @@ sub connect_path($scfg, $path) {
     });
 }
 
+sub parse_discovery_log($raw) {
+    my $log = decode_json($raw);
+    die "invalid NVMe discovery log\n"
+        if ref($log) ne 'HASH' || ref($log->{records}) ne 'ARRAY';
+    my (%seen, %res);
+    for my $record ($log->{records}->@*) {
+        next
+            if ($record->{subtype} // '') ne 'nvme subsystem' || ($record->{trtype} // '') ne 'tcp';
+        my $nqn = $record->{subnqn} // next;
+        next
+            if $nqn !~ /^$NQN_PATTERN$/
+            || length($nqn) > 223
+            || $nqn eq 'nqn.2014-08.org.nvmexpress.discovery';
+        my $addr = $record->{traddr} // '';
+        my $port = $record->{trsvcid} // '';
+        my $path = parse_portal(($addr =~ /:/ ? "[$addr]" : $addr) . ":$port") // next;
+        next if $seen{$nqn}->{ format_path($path) }++;
+        push $res{$nqn}->@*, $path;
+    }
+    return \%res;
+}
+
+sub discover($portal) {
+    my $path = parse_portal($portal) // die "invalid portal '$portal'\n";
+    my @ports = $portal =~ /:\d+$/ ? ($path->{trsvcid}) : (8009, 4420);
+    my $error;
+    for my $port (@ports) {
+        my $result = eval {
+            die "no answer from portal\n" if !PVE::Network::tcp_ping($path->{traddr}, $port, 2);
+            with_lock(sub {
+                assert_support();
+                my $host = host_identity();
+                my $out = '';
+                run_command(
+                    [
+                        $NVME,
+                        'discover',
+                        '--transport=tcp',
+                        '--traddr',
+                        $path->{traddr},
+                        '--trsvcid',
+                        $port,
+                        '--hostnqn',
+                        $host->{nqn},
+                        '--hostid',
+                        $host->{id},
+                        '--output-format=json',
+                    ],
+                    timeout => 30,
+                    outfunc => sub($line) { $out .= "$line\n" },
+                );
+                return parse_discovery_log($out);
+            });
+        };
+        $error = $@;
+        return $result if !$error;
+    }
+    die "discovery at portal '$portal' failed - $error";
+}
+
 1;
diff --git a/src/test/run_nvmeof_taint_tests.pl b/src/test/run_nvmeof_taint_tests.pl
index 3c19f8cc..d136c4f1 100755
--- a/src/test/run_nvmeof_taint_tests.pl
+++ b/src/test/run_nvmeof_taint_tests.pl
@@ -38,6 +38,7 @@ $nvme->mock(
             'all connect operands are validated and untainted',
         );
         PVE::Tools::run_command(['/usr/bin/true', @$cmd], timeout => 5);
+        $opts{outfunc}->('{"records":[]}') if $opts{outfunc};
     },
 );
 my $path = PVE::NVMeOF::parse_portal($portal);
@@ -53,6 +54,10 @@ for my $bad ('nqn.bad,nqn=other', "nqn.test\n", 'nqn.2014-08.org.nvmexpress.disc
     eval { PVE::NVMeOF::connect_path({ subsysnqn => $bad }, $path) };
     like($@, qr/invalid subsystem/, 'invalid connect subsystem refused');
 }
+my $net = Test::MockModule->new('PVE::Network');
+$net->mock(tcp_ping => sub { 1 });
+eval { PVE::NVMeOF::discover($portal) };
+is($@, '', 'discovery can execute with file-backed host identity in taint mode');
 file_set_contents($PVE::NVMeOF::HOSTID_FILE, "00000000-0000-0000-0000-000000000000\n");
 eval { PVE::NVMeOF::host_identity() };
 like($@, qr/nonzero UUID/, 'zero host ID refused');
diff --git a/src/test/run_nvmeof_tests.pl b/src/test/run_nvmeof_tests.pl
index 5ff96833..ead8d03c 100755
--- a/src/test/run_nvmeof_tests.pl
+++ b/src/test/run_nvmeof_tests.pl
@@ -9,6 +9,8 @@ use Test::More;
 use Test::MockModule;
 use PVE::NVMeOF;
 use PVE::Storage;
+use PVE::API2::Storage::Scan;
+use JSON qw(encode_json);
 use PVE::Tools qw(file_set_contents);
 
 my $dir = tempdir('nvmeof-test-XXXX', DIR => '.', CLEANUP => 1);
@@ -240,4 +242,56 @@ subtest 'local disk setup rejects fabric namespaces by transport' => sub {
     }
 };
 
+subtest 'TCP discovery, fallback and API result' => sub {
+    my $record = {
+        subtype => 'nvme subsystem',
+        trtype => 'tcp',
+        subnqn => 'nqn.test',
+        traddr => '192.0.2.1',
+        trsvcid => '4420',
+    };
+    my $raw = encode_json({
+        records =>
+            [$record, $record, { %$record, trtype => 'rdma' }, { %$record, trsvcid => 70000 }],
+    });
+    is_deeply(
+        PVE::NVMeOF::parse_discovery_log($raw),
+        { 'nqn.test' => [{ traddr => '192.0.2.1', trsvcid => 4420 }] },
+        'TCP only, unique valid portals',
+    );
+    for my $bad ('garbage', '{}', '[]') {
+        eval { PVE::NVMeOF::parse_discovery_log($bad) };
+        ok($@, 'malformed discovery is not an empty success');
+    }
+    my $nvme = Test::MockModule->new('PVE::NVMeOF');
+    my $net = Test::MockModule->new('PVE::Network');
+    my (@ports, @commands);
+    our $locked;
+    $nvme->mock(with_lock => sub { local $locked = 1; $_[0]->() });
+    $nvme->mock(assert_support => sub { });
+    $nvme->mock(host_identity => sub { { nqn => 'nqn.host', id => 'id' } });
+    $net->mock(tcp_ping => sub { push @ports, $_[1]; $_[1] != 8009 });
+    $nvme->mock(
+        run_command => sub($cmd, %opts) {
+            ok($locked, 'discovery holds the NVMe lock');
+            is($opts{timeout}, 30, 'discovery command timeout');
+            push @commands, $cmd;
+            $opts{outfunc}->($raw);
+        },
+    );
+    my $rows = PVE::API2::Storage::Scan->nvmeofscan({ node => 'n1', portal => '192.0.2.1' });
+    is_deeply(\@ports, [8009, 4420], 'unspecified port tries discovery then I/O port');
+    is_deeply($rows, [{ subsysnqn => 'nqn.test', portals => '192.0.2.1:4420' }], 'API shape');
+    eval {
+        PVE::JSONSchema::validate(
+            $rows,
+            PVE::API2::Storage::Scan->map_method_by_name('nvmeofscan')->{returns},
+        );
+    };
+    is($@, '', 'API schema accepts result');
+    @ports = ();
+    PVE::NVMeOF::discover('[2001:db8::1]:4421');
+    is_deeply(\@ports, [4421], 'explicit port does not fall back');
+};
+
 done_testing();




  parent reply	other threads:[~2026-11-13  3:25 UTC|newest]

Thread overview: 19+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-13  3:04 [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
2026-11-13  3:04 ` [PATCH storage 01/17] diskmanage: skip SMART queries for NVMe fabric namespaces Thomas Lamprecht
2026-11-13  3:04 ` [PATCH storage 02/17] nvmeof: add a storage for NVMe over Fabrics subsystems Thomas Lamprecht
2026-11-13  3:04 ` [PATCH storage 03/17] lvm: allow a volume group on an NVMe over Fabrics namespace Thomas Lamprecht
2026-11-13  3:04 ` Thomas Lamprecht [this message]
2026-11-13  3:04 ` [PATCH storage 05/17] pvesm: add a scan command for NVMe over Fabrics Thomas Lamprecht
2026-11-13  3:04 ` [PATCH storage 06/17] api: scan: san-luns: attribute NVMe namespaces to storage Thomas Lamprecht
2026-11-13  3:04 ` [PATCH storage 07/17] api: add node-level NVMe over Fabrics status and controller removal Thomas Lamprecht
2026-11-13  3:04 ` [PATCH storage 08/17] multipath: report the NVMe host NQN of each node Thomas Lamprecht
2026-11-13  3:04 ` [PATCH storage 09/17] multipath: report NVMe over Fabrics namespaces in the cluster status Thomas Lamprecht
2026-11-13  3:04 ` [PATCH docs 10/17] storage: document NVMe over Fabrics storages Thomas Lamprecht
2026-11-13  3:04 ` [PATCH manager 11/17] api: nodes: mount the NVMe over Fabrics endpoints Thomas Lamprecht
2026-11-13  3:04 ` [PATCH manager 12/17] ui: storage: add NVMe over Fabrics storage Thomas Lamprecht
2026-11-13  3:04 ` [PATCH manager 13/17] ui: lvm: offer NVMe over Fabrics storages as base storage Thomas Lamprecht
2026-11-13  3:04 ` [PATCH manager 14/17] ui: dc: san: add a guided setup for shared LVM on NVMe over Fabrics Thomas Lamprecht
2026-11-13  3:04 ` [PATCH manager 15/17] ui: node: san: list NVMe over Fabrics controllers Thomas Lamprecht
2026-11-13  3:04 ` [PATCH manager 16/17] ui: dc: san: show the NVMe host NQN of each node Thomas Lamprecht
2026-11-13  3:04 ` [PATCH manager 17/17] ui: dc: san: show that a namespace uses native multipath Thomas Lamprecht
2026-11-18 20:44 ` superseded: [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261113030541.1591528-5-t.lamprecht@proxmox.com \
    --to=t.lamprecht@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal