From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 09B611FF0B7 for ; Fri, 13 Nov 2026 04:33:07 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id C9D4821CDE; Fri, 13 Nov 2026 04:22:17 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH storage 07/17] api: add node-level NVMe over Fabrics status and controller removal Date: Fri, 13 Nov 2026 04:04:35 +0100 Message-ID: <20261113030541.1591528-8-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113030541.1591528-1-t.lamprecht@proxmox.com> References: <20261113030541.1591528-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794539148338 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.918 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: PPGPK7BAJUU7VX74WUPSPD4LFZRSVV2U X-Message-ID-Hash: PPGPK7BAJUU7VX74WUPSPD4LFZRSVV2U X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The host NQN for the access rules of an array and the controllers of a node are only visible with nvme-cli on a shell. Add the API module for /nodes/{node}/nvmeof with the host identity and the fabrics controllers with their subsystem, portal, state and storage. Removing a portal or the storage does not disconnect, so a controller can be removed here, unless a namespace in use would lose its last usable path. Reading needs Sys.Audit on the node or Datastore.Allocate on /storage, the removal Sys.Modify on '/'. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Makefile | 1 + src/PVE/API2/NVMeOF.pm | 182 +++++++++++++++++++++++++++++++ src/PVE/Diskmanage.pm | 16 +-- src/PVE/NVMeOF.pm | 41 +++++++ src/test/Makefile | 3 +- src/test/run_nvmeof_api_tests.pl | 151 +++++++++++++++++++++++++ 6 files changed, 385 insertions(+), 9 deletions(-) create mode 100644 src/PVE/API2/NVMeOF.pm create mode 100755 src/test/run_nvmeof_api_tests.pl diff --git a/src/PVE/API2/Makefile b/src/PVE/API2/Makefile index 9ab978b0..1be9ab04 100644 --- a/src/PVE/API2/Makefile +++ b/src/PVE/API2/Makefile @@ -4,6 +4,7 @@ install: install -D -m 0644 Disks.pm ${DESTDIR}${PERLDIR}/PVE/API2/Disks.pm install -D -m 0644 ISCSI.pm ${DESTDIR}${PERLDIR}/PVE/API2/ISCSI.pm + install -D -m 0644 NVMeOF.pm ${DESTDIR}${PERLDIR}/PVE/API2/NVMeOF.pm install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/API2/Multipath.pm make -C Storage install make -C Disks install diff --git a/src/PVE/API2/NVMeOF.pm b/src/PVE/API2/NVMeOF.pm new file mode 100644 index 00000000..57e70df4 --- /dev/null +++ b/src/PVE/API2/NVMeOF.pm @@ -0,0 +1,182 @@ +package PVE::API2::NVMeOF; + +use v5.36; + +use PVE::JSONSchema qw(get_standard_option); +use PVE::NVMeOF; +use PVE::RESTHandler; +use PVE::RPCEnvironment; +use PVE::Storage; + +use base qw(PVE::RESTHandler); + +my $read_permissions = { + check => [ + 'or', + ['perm', '/nodes/{node}', ['Sys.Audit']], + ['perm', '/storage', ['Datastore.Allocate']], + ], +}; +my $controller_schema = { + description => 'The fabrics controller name, for example nvme3.', + type => 'string', + pattern => 'nvme\d+', +}; + +__PACKAGE__->register_method({ + name => 'index', + path => '', + method => 'GET', + proxyto => 'node', + permissions => { user => 'all' }, + description => 'Node NVMe over Fabrics index.', + parameters => + { additionalProperties => 0, properties => { node => get_standard_option('pve-node') } }, + returns => { + type => 'array', + items => { type => 'object', properties => { name => { type => 'string' } } }, + links => [{ rel => 'child', href => '{name}' }], + }, + code => sub { return [{ name => 'status' }, { name => 'controller' }] }, +}); + +__PACKAGE__->register_method({ + name => 'status', + path => 'status', + method => 'GET', + protected => 1, + proxyto => 'node', + permissions => $read_permissions, + description => 'Report the NVMe host identity and nvme-cli availability.', + parameters => + { additionalProperties => 0, properties => { node => get_standard_option('pve-node') } }, + returns => { + type => 'object', + additionalProperties => 0, + properties => { + supported => { type => 'boolean', description => 'Whether nvme-cli is installed.' }, + 'host-nqn' => { + type => 'string', + optional => 1, + description => 'The host NQN for target access control.', + }, + 'host-id' => + { type => 'string', optional => 1, description => 'The persistent host UUID.' }, + error => { + type => 'string', + optional => 1, + description => 'Why the host identity is unavailable.', + }, + }, + }, + code => sub { + my $res = { supported => PVE::NVMeOF::is_supported() }; + my $host = eval { PVE::NVMeOF::host_identity() }; + if (my $error = $@) { + chomp $error; + $res->{error} = $error; + } else { + $res->{'host-nqn'} = $host->{nqn}; + $res->{'host-id'} = $host->{id}; + } + return $res; + }, +}); + +__PACKAGE__->register_method({ + name => 'controllers', + path => 'controller', + method => 'GET', + protected => 1, + proxyto => 'node', + permissions => $read_permissions, + description => 'List connected NVMe over Fabrics controllers and their states.', + parameters => + { additionalProperties => 0, properties => { node => get_standard_option('pve-node') } }, + returns => { + type => 'array', + items => { + type => 'object', + additionalProperties => 0, + properties => { + controller => $controller_schema, + subsysnqn => { type => 'string', description => 'The subsystem NQN.' }, + transport => { type => 'string', description => 'The controller transport.' }, + portal => { + type => 'string', + description => 'The controller address and port, if applicable.', + }, + state => { + type => 'string', + description => 'The controller state, such as live or connecting.', + }, + namespaces => { + type => 'integer', + description => 'The number of namespace devices in the subsystem.', + }, + 'used-by-storage' => { + type => 'string', + optional => 1, + description => 'An NVMe over Fabrics storage using the subsystem.', + }, + }, + }, + }, + code => sub { + my $cfg = PVE::Storage::config(); + my (%owners, %counts); + for my $sid (sort keys $cfg->{ids}->%*) { + my $scfg = $cfg->{ids}->{$sid}; + $owners{ $scfg->{subsysnqn} } //= $sid if $scfg->{type} eq 'nvmeof'; + } + my $controllers = PVE::NVMeOF::controllers(); + my @rows; + for my $name (sort keys %$controllers) { + my $ctrl = $controllers->{$name}; + my $nqn = $ctrl->{subsystem}; + $counts{$nqn} //= scalar(keys PVE::NVMeOF::namespaces($nqn)->%*); + push @rows, + { + controller => $name, + subsysnqn => $nqn, + transport => $ctrl->{transport}, + portal => PVE::NVMeOF::format_path($ctrl), + state => $ctrl->{state}, + namespaces => $counts{$nqn}, + defined($owners{$nqn}) ? ('used-by-storage' => $owners{$nqn}) : (), + }; + } + return \@rows; + }, +}); + +__PACKAGE__->register_method({ + name => 'disconnect', + path => 'controller/{controller}', + method => 'DELETE', + protected => 1, + proxyto => 'node', + permissions => { check => ['perm', '/', ['Sys.Modify']] }, + description => 'Disconnect one fabrics controller. Refuse when a namespace in use has no' + . ' other usable path. A storage still listing the portal reconnects it on activation.', + parameters => { + additionalProperties => 0, + properties => + { node => get_standard_option('pve-node'), controller => $controller_schema }, + }, + returns => { type => 'string', description => 'The task identifier.' }, + code => sub { + my ($param) = @_; + my $rpcenv = PVE::RPCEnvironment::get(); + return $rpcenv->fork_worker( + 'nvmedisconnect', + $param->{controller}, + $rpcenv->get_user(), + sub { + PVE::NVMeOF::disconnect_guarded($param->{controller}); + }, + ); + }, +}); + +1; diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm index 7bee3b08..3141ca7a 100644 --- a/src/PVE/Diskmanage.pm +++ b/src/PVE/Diskmanage.pm @@ -898,6 +898,13 @@ sub get_multipath_disks { return $disklist; } +sub nvme_path_is_usable { + my ($path) = @_; + my $ana = $path->{'ana-state'}; + return $path->{state} eq 'live' + && (!defined($ana) || $ana eq 'optimized' || $ana eq 'non-optimized'); +} + # Native multipath creates per-path namespace children nvmeXcYnZ below controllers. Local # single-controller devices lack these children and are omitted. sub get_nvme_path_status { @@ -933,15 +940,8 @@ sub get_nvme_path_status { my $ana = file_read_firstline("$ctrldir/$pathdev/ana_state"); $entry->{'ana-state'} = $ana if defined($ana); - # 'change' is transient; paths in 'inaccessible' or 'persistent-loss' - # cannot serve IO even on a live controller - my $usable = $state eq 'live' - && (!defined($ana) - || $ana eq 'optimized' - || $ana eq 'non-optimized'); - my $status = $res->{$head} //= { active => 0, paths => [] }; - $status->{active}++ if $usable; + $status->{active}++ if nvme_path_is_usable($entry); push @{ $status->{paths} }, $entry; }, ); diff --git a/src/PVE/NVMeOF.pm b/src/PVE/NVMeOF.pm index d1dfe15b..c81cb72f 100644 --- a/src/PVE/NVMeOF.pm +++ b/src/PVE/NVMeOF.pm @@ -308,4 +308,45 @@ sub discover($portal) { die "discovery at portal '$portal' failed - $error"; } +sub disconnect_guarded($controller) { + my ($name) = $controller =~ /\A(nvme\d+)\z/; + die "invalid controller name '$controller'\n" if !defined($name); + return with_lock(sub { + require PVE::Storage::SAN; + my $ctrl = controllers()->{$name} + // die "'$name' is not an NVMe over Fabrics controller\n"; + my $namespaces = namespaces($ctrl->{subsystem}); + my $status = PVE::Diskmanage::get_nvme_path_status(); + my %heads = map { $_ => 1 } keys %$namespaces; + for my $head (keys %$status) { + $heads{$head} = 1 + if grep { $_->{device} eq "/dev/$name" } $status->{$head}->{paths}->@*; + } + PVE::Cluster::cfs_update(1); + my $cfg = PVE::Storage::config(); + for my $head (sort keys %heads) { + my $paths = $status->{$head}->{paths} // []; + next + if grep { $_->{device} ne "/dev/$name" && PVE::Diskmanage::nvme_path_is_usable($_) } + @$paths; + my $ns = $namespaces->{$head} // die + "cannot disconnect '$name' - namespace '$head' has no device on this node to check for use\n"; + my $device = "/dev/$head"; + die + "cannot disconnect '$name' - namespace '$head' is in use and has no other usable path\n" + if PVE::Diskmanage::is_mounted($device) || PVE::Diskmanage::has_holder($device); + eval { + PVE::Storage::SAN::assert_lun_unreferenced( + $cfg, + { devpath => $device, wwid => $ns->{wwid} }, + ); + }; + die "cannot disconnect '$name' - $@" if $@; + } + die "nvme-cli is not installed\n" if !is_supported(); + run_command([$NVME, 'disconnect', '--device', "/dev/$name"], timeout => 30); + return; + }); +} + 1; diff --git a/src/test/Makefile b/src/test/Makefile index 124612da..73183dc1 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -77,6 +77,7 @@ test_san_api: run_san_api_tests.pl run_san_wipe_tests.pl run_san_nvme_tests.pl test_disk_api: run_disk_api_tests.pl ./run_disk_api_tests.pl -test_nvmeof: run_nvmeof_tests.pl run_nvmeof_taint_tests.pl +test_nvmeof: run_nvmeof_tests.pl run_nvmeof_api_tests.pl run_nvmeof_taint_tests.pl ./run_nvmeof_tests.pl + ./run_nvmeof_api_tests.pl perl -T ./run_nvmeof_taint_tests.pl diff --git a/src/test/run_nvmeof_api_tests.pl b/src/test/run_nvmeof_api_tests.pl new file mode 100755 index 00000000..3849dba5 --- /dev/null +++ b/src/test/run_nvmeof_api_tests.pl @@ -0,0 +1,151 @@ +#!/usr/bin/perl + +use v5.36; +use lib qw(..); +use Test::More; +use Test::MockModule; +use PVE::API2::NVMeOF; +use PVE::Storage::SAN; + +subtest 'last-path disconnect is checked inside serialization' => sub { + my $nvme = Test::MockModule->new('PVE::NVMeOF'); + my $disk = Test::MockModule->new('PVE::Diskmanage'); + my $storage = Test::MockModule->new('PVE::Storage'); + my $san = Test::MockModule->new('PVE::Storage::SAN'); + my $cluster = Test::MockModule->new('PVE::Cluster'); + our $locked; + my ($commands, $checks, $mounted, $busy, $headless) = (0, 0, 0, 0, 0); + my $other = { device => '/dev/nvme2', state => 'live', 'ana-state' => 'optimized' }; + $nvme->mock(with_lock => + sub { die "NVMe over Fabrics is busy\n" if $busy; local $locked = 1; $_[0]->() }); + $nvme->mock( + controllers => sub { + ok($locked, 'controller snapshot is locked'); + { nvme1 => { subsystem => 'nqn.test' } }; + }, + ); + $nvme->mock(namespaces => sub { $headless ? {} : { nvme0n1 => { wwid => 'eui.abc' } } }); + $nvme->mock(is_supported => sub { 1 }); + $disk->mock( + get_nvme_path_status => + sub { { + nvme0n1 => { paths => [{ device => '/dev/nvme1', state => 'live' }, $other] } } }, + ); + $disk->mock(is_mounted => sub { $mounted }); + $disk->mock(has_holder => sub { 0 }); + $cluster->mock(cfs_update => sub { ok($locked, 'configuration refreshed under lock') }); + $storage->mock(config => sub { { ids => {} } }); + $san->mock( + assert_lun_unreferenced => sub { $checks++; die "storage uses this namespace\n" }); + $nvme->mock( + run_command => sub($cmd, %opts) { + ok($locked, 'deletion keeps the lock'); + is($opts{timeout}, 30, 'deletion is bounded'); + is_deeply( + $cmd, + ['/usr/sbin/nvme', 'disconnect', '--device', '/dev/nvme1'], + 'only the selected controller is removed', + ); + $commands++; + }, + ); + for my $ana ('optimized', 'non-optimized') { + $other->{'ana-state'} = $ana; + PVE::NVMeOF::disconnect_guarded('nvme1'); + } + is($commands, 2, 'another usable path permits removal'); + is($checks, 0, 'no usage probe when another path is usable'); + for my $ana ('inaccessible', 'persistent-loss', 'change') { + $other->{'ana-state'} = $ana; + eval { PVE::NVMeOF::disconnect_guarded('nvme1') }; + like($@, qr/storage uses this namespace/, "$ana does not protect an in-use namespace"); + } + $other = { device => '/dev/nvme2', state => 'connecting', 'ana-state' => 'optimized' }; + eval { PVE::NVMeOF::disconnect_guarded('nvme1') }; + like($@, qr/storage uses this namespace/, 'reconnecting is not a usable path'); + is($commands, 2, 'refusals never delete'); + $san->mock(assert_lun_unreferenced => sub { $checks++ }); + $mounted = 1; + eval { PVE::NVMeOF::disconnect_guarded('nvme1') }; + like($@, qr/is in use/, 'local mount refuses'); + $mounted = 0; + PVE::NVMeOF::disconnect_guarded('nvme1'); + is($commands, 3, 'unused namespace can lose its final path'); + $headless = 1; + eval { PVE::NVMeOF::disconnect_guarded('nvme1') }; + like($@, qr/has no device/, 'uncheckable namespace refuses'); + $busy = 1; + $nvme->mock(controllers => sub { die "must not count paths\n" }); + eval { PVE::NVMeOF::disconnect_guarded('nvme1') }; + like($@, qr/is busy/, 'ongoing deletion blocks another snapshot'); + eval { PVE::NVMeOF::disconnect_guarded('../nvme1') }; + like($@, qr/invalid controller/, 'invalid controller refused before locking'); +}; + +subtest 'node API contract' => sub { + my $api = 'PVE::API2::NVMeOF'; + my $nvme = Test::MockModule->new('PVE::NVMeOF'); + my $storage = Test::MockModule->new('PVE::Storage'); + $storage->mock( + config => + sub { { ids => { fabric => { type => 'nvmeof', subsysnqn => 'nqn.test' } } } }, + ); + $nvme->mock(is_supported => sub { 0 }); + $nvme->mock(host_identity => sub { { nqn => 'nqn.host', id => 'uuid' } }); + my $status = $api->status({ node => 'n1' }); + is($status->{supported}, 0, 'availability means nvme-cli, not identity or version'); + is($status->{'host-nqn'}, 'nqn.host', 'host identity shown independently'); + $nvme->mock(is_supported => sub { 1 }); + $nvme->mock(host_identity => sub { die "missing host identity\n" }); + $status = $api->status({ node => 'n1' }); + is_deeply( + $status, + { supported => 1, error => 'missing host identity' }, + 'identity error does not hide installed tools', + ); + $nvme->mock( + controllers => sub { { + nvme1 => { + subsystem => 'nqn.test', + transport => 'tcp', + traddr => '192.0.2.1', + trsvcid => 4420, + state => 'connecting', + }, + } }, + ); + $nvme->mock(namespaces => sub { { nvme0n1 => {} } }); + my $rows = $api->controllers({ node => 'n1' }); + is($rows->[0]->{'used-by-storage'}, 'fabric', 'controller attributed'); + is($rows->[0]->{state}, 'connecting', 'kernel state remains visible'); + + for my $pair (['status', $status], ['controllers', $rows]) { + my $method = $api->map_method_by_name($pair->[0]); + eval { PVE::JSONSchema::validate($pair->[1], $method->{returns}) }; + is($@, '', "$pair->[0] schema"); + ok($method->{protected} && $method->{proxyto} eq 'node', 'protected and node-proxied'); + } + is( + $api->map_method_by_name('disconnect')->{path}, + 'controller/{controller}', + 'REST-shaped deletion', + ); + my $rpc = Test::MockModule->new('PVE::RPCEnvironment'); + { + + package NVMeTestRPC; + sub get_user { 'root@pam' } + sub fork_worker { $_[4]->(); return 'UPID:test' } + } + $rpc->mock(get => sub { bless {}, 'NVMeTestRPC' }); + my $removed; + $nvme->mock(disconnect_guarded => sub { $removed = $_[0] }); + is( + $api->disconnect({ node => 'n1', controller => 'nvme1' }), + 'UPID:test', + 'API returns task identifier', + ); + is($removed, 'nvme1', 'task calls guarded disconnect'); +}; + +done_testing();