all lists on lists.proxmox.com
 help / color / mirror / Atom feed
From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v2 storage 04/17] api: scan: add NVMe over Fabrics discovery
Date: Wed, 18 Nov 2026 21:42:23 +0100	[thread overview]
Message-ID: <20261118204302.3100813-5-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261118204302.3100813-1-t.lamprecht@proxmox.com>

Setting up an NVMe over Fabrics storage needs the NQN of the
subsystem and its portals. Add /nodes/{node}/scan/nvmeof, which asks
the discovery service at a portal for the NVMe/TCP subsystems it
offers to the node and lists them with their portals.

Without a port, it tries the discovery port 8009 and then the I/O
port 4420, where Linux targets answer discovery too. The scan needs
Datastore.Allocate on /storage, like the iSCSI scan.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 src/PVE/API2/Storage/Scan.pm       | 49 ++++++++++++++++++++++
 src/PVE/NVMeOF.pm                  | 65 ++++++++++++++++++++++++++++++
 src/test/run_nvmeof_taint_tests.pl |  5 +++
 src/test/run_nvmeof_tests.pl       | 54 +++++++++++++++++++++++++
 4 files changed, 173 insertions(+)

diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm
index cf19f835..84848847 100644
--- a/src/PVE/API2/Storage/Scan.pm
+++ b/src/PVE/API2/Storage/Scan.pm
@@ -6,6 +6,7 @@ use warnings;
 # NOTE: This API endpoints are mounted by pve-manager's API2::Node module and pvesm CLI
 
 use PVE::JSONSchema qw(get_standard_option);
+use PVE::NVMeOF;
 use PVE::RESTHandler;
 use PVE::RPCEnvironment;
 use PVE::SafeSyslog;
@@ -50,6 +51,7 @@ __PACKAGE__->register_method({
             { method => 'iscsi' },
             { method => 'lvm' },
             { method => 'nfs' },
+            { method => 'nvmeof' },
             { method => 'pbs' },
             { method => 'san-luns' },
             { method => 'zfs' },
@@ -302,6 +304,53 @@ __PACKAGE__->register_method({
     },
 });
 
+__PACKAGE__->register_method({
+    name => 'nvmeofscan',
+    path => 'nvmeof',
+    method => 'GET',
+    description =>
+        'List the NVMe over TCP subsystems offered to this node by a discovery controller.',
+    protected => 1,
+    proxyto => 'node',
+    permissions => { check => ['perm', '/storage', ['Datastore.Allocate']] },
+    parameters => {
+        additionalProperties => 0,
+        properties => {
+            node => get_standard_option('pve-node'),
+            portal => {
+                description => 'IPv4 or bracketed IPv6 discovery address with an optional port.'
+                    . ' Without a port, try 8009 then 4420.',
+                type => 'string',
+                format => 'pve-storage-nvme-portal',
+            },
+        },
+    },
+    returns => {
+        type => 'array',
+        items => {
+            type => 'object',
+            additionalProperties => 0,
+            properties => {
+                subsysnqn => { type => 'string', description => 'The subsystem NQN.' },
+                portals => {
+                    type => 'string',
+                    description => 'The discovered TCP portals, comma separated.',
+                },
+            },
+        },
+    },
+    code => sub {
+        my ($param) = @_;
+        my $res = PVE::NVMeOF::discover($param->{portal});
+        return [
+            map { {
+                subsysnqn => $_,
+                portals => join(',', map { PVE::NVMeOF::format_path($_) } $res->{$_}->@*),
+            } } sort keys %$res
+        ];
+    },
+});
+
 __PACKAGE__->register_method({
     name => 'lvmscan',
     path => 'lvm',
diff --git a/src/PVE/NVMeOF.pm b/src/PVE/NVMeOF.pm
index ebaedf9b..f9a293d9 100644
--- a/src/PVE/NVMeOF.pm
+++ b/src/PVE/NVMeOF.pm
@@ -5,9 +5,11 @@ use v5.36;
 use Digest::SHA qw(sha1_hex);
 use Fcntl qw(:flock F_GETFD F_SETFD FD_CLOEXEC);
 use File::Path qw(make_path);
+use JSON qw(decode_json);
 use Socket qw(AF_INET AF_INET6 inet_ntop inet_pton);
 
 use PVE::JSONSchema;
+use PVE::Network;
 use PVE::Tools qw(dir_glob_foreach file_read_firstline file_set_contents run_command);
 
 use PVE::Diskmanage;
@@ -241,4 +243,67 @@ sub connect_path($scfg, $path) {
     });
 }
 
+sub parse_discovery_log($raw) {
+    my $log = decode_json($raw);
+    die "invalid NVMe discovery log\n"
+        if ref($log) ne 'HASH' || ref($log->{records}) ne 'ARRAY';
+    my (%seen, %res);
+    for my $record ($log->{records}->@*) {
+        next
+            if ($record->{subtype} // '') ne 'nvme subsystem' || ($record->{trtype} // '') ne 'tcp';
+        my $nqn = $record->{subnqn} // next;
+        if (
+            $nqn !~ /^$NQN_PATTERN$/
+            || length($nqn) > 223
+            || $nqn eq 'nqn.2014-08.org.nvmexpress.discovery'
+        ) {
+            next;
+        }
+        my $addr = $record->{traddr} // '';
+        my $port = $record->{trsvcid} // '';
+        my $path = parse_portal(($addr =~ /:/ ? "[$addr]" : $addr) . ":$port") // next;
+        next if $seen{$nqn}->{ format_path($path) }++;
+        push $res{$nqn}->@*, $path;
+    }
+    return \%res;
+}
+
+sub discover($portal) {
+    my $path = parse_portal($portal) // die "invalid portal '$portal'\n";
+    my @ports = $portal =~ /:[0-9]+$/ ? ($path->{trsvcid}) : (8009, 4420);
+    my $error;
+    for my $port (@ports) {
+        my $result = eval {
+            die "no answer from portal\n" if !PVE::Network::tcp_ping($path->{traddr}, $port, 2);
+            with_lock(sub {
+                assert_support();
+                my $host = host_identity();
+                my $out = '';
+                run_command(
+                    [
+                        $NVME,
+                        'discover',
+                        '--transport=tcp',
+                        '--traddr',
+                        $path->{traddr},
+                        '--trsvcid',
+                        $port,
+                        '--hostnqn',
+                        $host->{nqn},
+                        '--hostid',
+                        $host->{id},
+                        '--output-format=json',
+                    ],
+                    timeout => 30,
+                    outfunc => sub($line) { $out .= "$line\n" },
+                );
+                return parse_discovery_log($out);
+            });
+        };
+        $error = $@;
+        return $result if !$error;
+    }
+    die "discovery at portal '$portal' failed - $error";
+}
+
 1;
diff --git a/src/test/run_nvmeof_taint_tests.pl b/src/test/run_nvmeof_taint_tests.pl
index 62405291..c6546c30 100755
--- a/src/test/run_nvmeof_taint_tests.pl
+++ b/src/test/run_nvmeof_taint_tests.pl
@@ -38,6 +38,7 @@ $nvme->mock(
             'all connect operands are validated and untainted',
         );
         PVE::Tools::run_command(['/usr/bin/true', @$cmd], timeout => 5);
+        $opts{outfunc}->('{"records":[]}') if $opts{outfunc};
     },
 );
 my $path = PVE::NVMeOF::parse_portal($portal);
@@ -53,6 +54,10 @@ for my $bad ('nqn.bad,nqn=other', "nqn.test\n", 'nqn.2014-08.org.nvmexpress.disc
     eval { PVE::NVMeOF::connect_path({ subsysnqn => $bad }, $path) };
     like($@, qr/invalid subsystem/, 'invalid connect subsystem refused');
 }
+my $net = Test::MockModule->new('PVE::Network');
+$net->mock(tcp_ping => sub { 1 });
+eval { PVE::NVMeOF::discover($portal) };
+is($@, '', 'discovery can execute with file-backed host identity in taint mode');
 file_set_contents($PVE::NVMeOF::HOSTID_FILE, "00000000-0000-0000-0000-000000000000\n");
 eval { PVE::NVMeOF::host_identity() };
 like($@, qr/nonzero UUID/, 'zero host ID refused');
diff --git a/src/test/run_nvmeof_tests.pl b/src/test/run_nvmeof_tests.pl
index b75f7a5a..ffbaa45f 100755
--- a/src/test/run_nvmeof_tests.pl
+++ b/src/test/run_nvmeof_tests.pl
@@ -5,6 +5,7 @@ use lib qw(..);
 use Fcntl qw(:flock);
 use File::Path qw(make_path);
 use File::Temp qw(tempdir);
+use JSON qw(encode_json);
 use Test::MockModule;
 use Test::More;
 
@@ -12,6 +13,7 @@ use PVE::Tools qw(file_set_contents);
 
 use PVE::NVMeOF;
 use PVE::Storage;
+use PVE::API2::Storage::Scan;
 
 my $dir = tempdir('nvmeof-test-XXXX', DIR => '.', CLEANUP => 1);
 local $PVE::NVMeOF::RUN_DIR = "$dir/run";
@@ -299,4 +301,56 @@ subtest 'local disk setup rejects fabric namespaces by transport' => sub {
     }
 };
 
+subtest 'TCP discovery, fallback and API result' => sub {
+    my $record = {
+        subtype => 'nvme subsystem',
+        trtype => 'tcp',
+        subnqn => 'nqn.test',
+        traddr => '192.0.2.1',
+        trsvcid => '4420',
+    };
+    my $raw = encode_json({
+        records =>
+            [$record, $record, { %$record, trtype => 'rdma' }, { %$record, trsvcid => 70000 }],
+    });
+    is_deeply(
+        PVE::NVMeOF::parse_discovery_log($raw),
+        { 'nqn.test' => [{ traddr => '192.0.2.1', trsvcid => 4420 }] },
+        'TCP only, unique valid portals',
+    );
+    for my $bad ('garbage', '{}', '[]') {
+        eval { PVE::NVMeOF::parse_discovery_log($bad) };
+        ok($@, 'malformed discovery is not an empty success');
+    }
+    my $nvme = Test::MockModule->new('PVE::NVMeOF');
+    my $net = Test::MockModule->new('PVE::Network');
+    my (@ports, @commands);
+    our $locked;
+    $nvme->mock(with_lock => sub { local $locked = 1; $_[0]->() });
+    $nvme->mock(assert_support => sub { });
+    $nvme->mock(host_identity => sub { { nqn => 'nqn.host', id => 'id' } });
+    $net->mock(tcp_ping => sub { push @ports, $_[1]; $_[1] != 8009 });
+    $nvme->mock(
+        run_command => sub($cmd, %opts) {
+            ok($locked, 'discovery holds the NVMe lock');
+            is($opts{timeout}, 30, 'discovery command timeout');
+            push @commands, $cmd;
+            $opts{outfunc}->($raw);
+        },
+    );
+    my $rows = PVE::API2::Storage::Scan->nvmeofscan({ node => 'n1', portal => '192.0.2.1' });
+    is_deeply(\@ports, [8009, 4420], 'unspecified port tries discovery then I/O port');
+    is_deeply($rows, [{ subsysnqn => 'nqn.test', portals => '192.0.2.1:4420' }], 'API shape');
+    eval {
+        PVE::JSONSchema::validate(
+            $rows,
+            PVE::API2::Storage::Scan->map_method_by_name('nvmeofscan')->{returns},
+        );
+    };
+    is($@, '', 'API schema accepts result');
+    @ports = ();
+    PVE::NVMeOF::discover('[2001:db8::1]:4421');
+    is_deeply(\@ports, [4421], 'explicit port does not fall back');
+};
+
 done_testing();




  parent reply	other threads:[~2026-11-18 20:43 UTC|newest]

Thread overview: 18+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-18 20:42 [PATCH v2 storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 storage 01/17] diskmanage: skip SMART queries for NVMe fabric namespaces Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 storage 02/17] nvmeof: add a storage for NVMe over Fabrics subsystems Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 storage 03/17] lvm: allow a volume group on an NVMe over Fabrics namespace Thomas Lamprecht
2026-11-18 20:42 ` Thomas Lamprecht [this message]
2026-11-18 20:42 ` [PATCH v2 storage 05/17] pvesm: add a scan command for NVMe over Fabrics Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 storage 06/17] api: scan: san-luns: attribute NVMe namespaces to storage Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 storage 07/17] api: add node-level NVMe over Fabrics status and controller removal Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 storage 08/17] multipath: report the NVMe host NQN of each node Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 storage 09/17] multipath: report NVMe over Fabrics namespaces in the cluster status Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 docs 10/17] storage: document NVMe over Fabrics storages Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 manager 11/17] api: nodes: mount the NVMe over Fabrics endpoints Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 manager 12/17] ui: storage: add NVMe over Fabrics storage Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 manager 13/17] ui: lvm: offer NVMe over Fabrics storages as base storage Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 manager 14/17] ui: dc: san: add a guided setup for shared LVM on NVMe over Fabrics Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 manager 15/17] ui: node: san: list NVMe over Fabrics controllers Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 manager 16/17] ui: dc: san: show the NVMe host NQN of each node Thomas Lamprecht
2026-11-18 20:42 ` [PATCH v2 manager 17/17] ui: dc: san: show that a namespace uses native multipath Thomas Lamprecht

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261118204302.3100813-5-t.lamprecht@proxmox.com \
    --to=t.lamprecht@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal