From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH storage 02/17] nvmeof: add a storage for NVMe over Fabrics subsystems
Date: Fri, 13 Nov 2026 04:04:30 +0100 [thread overview]
Message-ID: <20261113030541.1591528-3-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261113030541.1591528-1-t.lamprecht@proxmox.com>
NVMe/TCP namespaces need a manual nvme-cli setup on every node so
far. Add a storage type that connects one subsystem through each
configured portal and offers its namespaces as raw volumes. Native
NVMe multipath joins the paths and the kernel reconnects lost ones.
Local disk setup refuses namespaces attached over a fabric, as for
iSCSI LUNs. Wiping a namespace or creating a volume group on it is
refused while a storage serves images from its subsystem. nvme-cli
is only recommended, as no other storage type needs it.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
debian/control | 1 +
src/PVE/Diskmanage.pm | 10 ++
src/PVE/Makefile | 1 +
src/PVE/NVMeOF.pm | 249 +++++++++++++++++++++++++++++
src/PVE/Storage.pm | 2 +
src/PVE/Storage/Makefile | 1 +
src/PVE/Storage/NVMeOFPlugin.pm | 204 +++++++++++++++++++++++
src/PVE/Storage/Plugin.pm | 2 +-
src/PVE/Storage/SAN.pm | 13 ++
src/test/Makefile | 6 +-
src/test/run_nvmeof_taint_tests.pl | 60 +++++++
src/test/run_nvmeof_tests.pl | 243 ++++++++++++++++++++++++++++
src/test/run_san_nvme_tests.pl | 13 ++
13 files changed, 803 insertions(+), 2 deletions(-)
create mode 100644 src/PVE/NVMeOF.pm
create mode 100644 src/PVE/Storage/NVMeOFPlugin.pm
create mode 100755 src/test/run_nvmeof_taint_tests.pl
create mode 100755 src/test/run_nvmeof_tests.pl
diff --git a/debian/control b/debian/control
index 14d8a648..99b41f92 100644
--- a/debian/control
+++ b/debian/control
@@ -55,6 +55,7 @@ Depends: bzip2,
${misc:Depends},
${perl:Depends},
Recommends: multipath-tools,
+ nvme-cli,
pve-esxi-import-tools (>= 0.6.0),
skopeo,
zfsutils-linux (>= 2.4.1-pve2),
diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm
index 1fec9ec5..7bee3b08 100644
--- a/src/PVE/Diskmanage.pm
+++ b/src/PVE/Diskmanage.pm
@@ -108,6 +108,8 @@ sub disk_is_used {
my $disklist = get_disks($dev, 1, 1);
die "'$disk' is not a valid local disk\n" if !defined($disklist->{$dev});
+ die "'$disk' is not a valid local disk\n"
+ if ($disklist->{$dev}->{transport} // '') =~ /^nvme-(?:tcp|rdma|fc)$/;
return 1 if $disklist->{$dev}->{used};
return 0;
@@ -503,6 +505,14 @@ sub dir_is_empty {
return 1;
}
+# Some controllers pad the NQN or derive it from padded serial/model fields.
+sub read_subsysnqn {
+ my ($path) = @_;
+ my $nqn = file_read_firstline($path) // return undef;
+ $nqn =~ s/\s+$//;
+ return $nqn;
+}
+
# Returns the NVMe transport (pcie, fc, tcp, rdma, loop) of a namespace, either directly from
# its controller, or from any controller of its subsystem for native NVMe multipath setups.
sub get_nvme_transport {
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index 13fecbf8..238eefd4 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -4,6 +4,7 @@
install:
install -D -m 0644 Storage.pm ${DESTDIR}${PERLDIR}/PVE/Storage.pm
install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
+ install -D -m 0644 NVMeOF.pm ${DESTDIR}${PERLDIR}/PVE/NVMeOF.pm
install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm
install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm
install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm
diff --git a/src/PVE/NVMeOF.pm b/src/PVE/NVMeOF.pm
new file mode 100644
index 00000000..7a3140cb
--- /dev/null
+++ b/src/PVE/NVMeOF.pm
@@ -0,0 +1,249 @@
+package PVE::NVMeOF;
+
+use v5.36;
+
+use Digest::SHA qw(sha1_hex);
+use Fcntl qw(:flock F_GETFD F_SETFD FD_CLOEXEC);
+use File::Path qw(make_path);
+use Socket qw(AF_INET AF_INET6 inet_ntop inet_pton);
+
+use PVE::Diskmanage;
+use PVE::JSONSchema;
+use PVE::Tools qw(dir_glob_foreach file_read_firstline file_set_contents run_command);
+
+our $NVME = '/usr/sbin/nvme';
+our $SYSFS = '/sys';
+our $HOSTNQN_FILE = '/etc/nvme/hostnqn';
+our $HOSTID_FILE = '/etc/nvme/hostid';
+our $RUN_DIR = '/run/pve/nvmeof';
+our $NQN_PATTERN = 'nqn\.[^\s,=]+';
+
+sub canonical_address($addr) {
+ if (defined(my $packed = inet_pton(AF_INET, $addr))) {
+ return inet_ntop(AF_INET, $packed);
+ }
+ if (defined(my $packed = inet_pton(AF_INET6, $addr))) {
+ return inet_ntop(AF_INET, substr($packed, 12))
+ if substr($packed, 0, 12) eq ("\0" x 10) . "\xff\xff";
+ return inet_ntop(AF_INET6, $packed);
+ }
+ return lc($addr);
+}
+
+# Link-local IPv6 needs a node-specific interface and cannot identify a shared configured portal.
+sub parse_portal($portal) {
+ my ($addr, $port);
+ if ($portal =~ m/^\[([0-9a-fA-F:.]+)\](?::(\d+))?$/) {
+ ($addr, $port) = ($1, $2);
+ } elsif ($portal =~ m/^(\d+\.\d+\.\d+\.\d+)(?::(\d+))?$/) {
+ ($addr, $port) = ($1, $2);
+ } else {
+ return undef;
+ }
+ return undef if defined($port) && ($port < 1 || $port > 65535);
+ $addr = canonical_address($addr);
+ if (defined(my $packed = inet_pton(AF_INET, $addr))) {
+ return undef
+ if $addr eq '0.0.0.0'
+ || $addr eq '255.255.255.255'
+ || (unpack('C', $packed) & 0xf0) == 0xe0;
+ } elsif (defined($packed = inet_pton(AF_INET6, $addr))) {
+ my $prefix = unpack('n', $packed);
+ return undef
+ if $addr eq '::' || ($prefix & 0xffc0) == 0xfe80 || ($prefix & 0xff00) == 0xff00;
+ } else {
+ return undef;
+ }
+ return { traddr => $addr, trsvcid => int($port // 4420) };
+}
+
+PVE::JSONSchema::register_format(
+ 'pve-storage-nvme-portal',
+ sub($portal, $noerr = undef) {
+ return $portal if parse_portal($portal);
+ return undef if $noerr;
+ die "value is not an IPv4 address or a bracketed IPv6 address with optional port\n";
+ },
+);
+
+sub format_path($path) {
+ my $addr = $path->{traddr} =~ /:/ ? "[$path->{traddr}]" : $path->{traddr};
+ return defined($path->{trsvcid}) ? "$addr:$path->{trsvcid}" : $addr;
+}
+
+sub storage_paths($scfg) {
+ return [map { parse_portal($_) // die "invalid portal '$_'\n" }
+ PVE::Tools::split_list($scfg->{'nvme-portals'})];
+}
+
+sub host_identity() {
+ my ($nqn) = (file_read_firstline($HOSTNQN_FILE) // '') =~ /\A($NQN_PATTERN)\z/;
+ my ($id) = (file_read_firstline($HOSTID_FILE) // '') =~
+ /\A([0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})\z/i;
+ die
+ "invalid or missing NVMe host NQN in '$HOSTNQN_FILE' - install nvme-cli and check the host identity\n"
+ if !defined($nqn) || length($nqn) > 223;
+ die "invalid or missing NVMe host ID in '$HOSTID_FILE' - expected a nonzero UUID\n"
+ if !defined($id) || $id =~ /^[0-]+$/;
+ return { nqn => $nqn, id => lc($id) };
+}
+
+sub is_supported() { return -x $NVME ? 1 : 0 }
+
+sub assert_support() {
+ die "nvme-cli is not installed\n" if !is_supported();
+ run_command(['modprobe', 'nvme-tcp'], timeout => 10)
+ if !-e "$SYSFS/module/nvme_tcp";
+ my $multipath = file_read_firstline("$SYSFS/module/nvme_core/parameters/multipath") // '';
+ die "native NVMe multipath is disabled - enable nvme_core.multipath\n" if $multipath ne 'Y';
+}
+
+sub controllers() {
+ my $res = {};
+ my $dir = "$SYSFS/class/nvme-fabrics/ctl";
+ dir_glob_foreach(
+ $dir,
+ 'nvme\d+',
+ sub($name, @) {
+ my $address = file_read_firstline("$dir/$name/address") // return;
+ my %address = map { split(/=/, $_, 2) } grep { /=/ } split(/,/, $address);
+ return if !defined($address{traddr});
+ $res->{$name} = {
+ transport => file_read_firstline("$dir/$name/transport") // '',
+ subsystem => PVE::Diskmanage::read_subsysnqn("$dir/$name/subsysnqn") // '',
+ state => file_read_firstline("$dir/$name/state") // 'unknown',
+ traddr => canonical_address($address{traddr}),
+ trsvcid => $address{trsvcid},
+ };
+ },
+ );
+ return $res;
+}
+
+sub subsystem_controllers($subsystem) {
+ my $controllers = controllers();
+ return {
+ map { $_ => $controllers->{$_} } grep {
+ $controllers->{$_}->{subsystem} eq $subsystem
+ && $controllers->{$_}->{transport} eq 'tcp'
+ } keys %$controllers
+ };
+}
+
+sub path_controller($controllers, $path) {
+ for my $name (sort keys %$controllers) {
+ my $ctrl = $controllers->{$name};
+ return $name
+ if $ctrl->{traddr} eq $path->{traddr}
+ && ($ctrl->{trsvcid} // '') eq $path->{trsvcid};
+ }
+ return undef;
+}
+
+sub namespaces($subsystem) {
+ my $res = {};
+ my $dir = "$SYSFS/class/nvme-subsystem";
+ dir_glob_foreach(
+ $dir,
+ 'nvme-subsys\d+',
+ sub($subsys, @) {
+ my $nqn = PVE::Diskmanage::read_subsysnqn("$dir/$subsys/subsysnqn") // return;
+ return if $nqn ne $subsystem;
+ dir_glob_foreach(
+ "$dir/$subsys",
+ 'nvme\d+n\d+',
+ sub($dev, @) {
+ my $wwid = file_read_firstline("$SYSFS/block/$dev/wwid") // return;
+ my $size = file_read_firstline("$SYSFS/block/$dev/size") // return;
+ $res->{$dev} = {
+ nsid => int(file_read_firstline("$SYSFS/block/$dev/nsid") // 0),
+ wwid => $wwid,
+ size => int($size) * 512,
+ };
+ },
+ );
+ },
+ );
+ return $res;
+}
+
+sub with_lock($code) {
+ my ($res, $timeout) = PVE::Tools::run_fork_with_timeout(
+ 35,
+ sub {
+ make_path($RUN_DIR);
+ open(my $lock, '>>', "$RUN_DIR/fabrics.lock") or die "cannot open NVMe lock - $!\n";
+ flock($lock, LOCK_EX | LOCK_NB)
+ or die "NVMe over Fabrics is busy - another operation is still running\n";
+ # A command still in the kernel must retain serialization after its caller times out.
+ my $flags = fcntl($lock, F_GETFD, 0) // die "cannot read NVMe lock flags - $!\n";
+ fcntl($lock, F_SETFD, $flags & ~FD_CLOEXEC)
+ or die "cannot inherit NVMe lock - $!\n";
+ return $code->();
+ },
+ );
+ die "NVMe over Fabrics operation timed out - it may still be running\n" if $timeout;
+ return $res;
+}
+
+my sub retry_file($scfg, $path) {
+ my ($key) = sha1_hex(join("\0",
+ $scfg->{subsysnqn},
+ format_path($path),
+ $scfg->{'nvme-fast-io-fail-tmo'} // 120)) =~ /\A([0-9a-f]{40})\z/;
+ return "$RUN_DIR/retry/$key";
+}
+
+sub in_cooldown($scfg, $path) {
+ my $mtime = (stat(retry_file($scfg, $path)))[9] // return 0;
+ my $age = time() - $mtime;
+ return $age > -60 && $age < 60;
+}
+
+sub note_connect_failure($scfg, $path) {
+ make_path("$RUN_DIR/retry");
+ file_set_contents(retry_file($scfg, $path), '');
+}
+
+sub connect_path($scfg, $path) {
+ my ($nqn) = $scfg->{subsysnqn} =~ /\A($NQN_PATTERN)\z/;
+ die "invalid subsystem NQN\n"
+ if !defined($nqn)
+ || length($nqn) > 223
+ || $nqn eq 'nqn.2014-08.org.nvmexpress.discovery';
+ my ($fail) = ($scfg->{'nvme-fast-io-fail-tmo'} // 120) =~ /\A(-?\d+)\z/;
+ die "invalid fast I/O fail timeout\n" if !defined($fail) || $fail < -1;
+ return with_lock(sub {
+ return if path_controller(subsystem_controllers($scfg->{subsysnqn}), $path);
+ assert_support();
+ my $host = host_identity();
+ run_command(
+ [
+ $NVME,
+ 'connect',
+ '--transport=tcp',
+ '--nqn',
+ $nqn,
+ '--traddr',
+ $path->{traddr},
+ '--trsvcid',
+ $path->{trsvcid},
+ '--hostnqn',
+ $host->{nqn},
+ '--hostid',
+ $host->{id},
+ '--ctrl-loss-tmo=-1',
+ '--fast_io_fail_tmo',
+ $fail,
+ ],
+ timeout => 30,
+ );
+ # nvme-cli can exit successfully without having connected
+ die "no controller appeared\n"
+ if !path_controller(subsystem_controllers($scfg->{subsysnqn}), $path);
+ unlink(retry_file($scfg, $path));
+ return;
+ });
+}
+
+1;
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index be6e9743..287d64f0 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -32,6 +32,7 @@ use PVE::Storage::LvmThinPlugin;
use PVE::Storage::NFSPlugin;
use PVE::Storage::CIFSPlugin;
use PVE::Storage::ISCSIPlugin;
+use PVE::Storage::NVMeOFPlugin;
use PVE::Storage::RBDPlugin;
use PVE::Storage::CephFSPlugin;
use PVE::Storage::ISCSIDirectPlugin;
@@ -57,6 +58,7 @@ PVE::Storage::LvmThinPlugin->register();
PVE::Storage::NFSPlugin->register();
PVE::Storage::CIFSPlugin->register();
PVE::Storage::ISCSIPlugin->register();
+PVE::Storage::NVMeOFPlugin->register();
PVE::Storage::RBDPlugin->register();
PVE::Storage::CephFSPlugin->register();
PVE::Storage::ISCSIDirectPlugin->register();
diff --git a/src/PVE/Storage/Makefile b/src/PVE/Storage/Makefile
index a2df83c0..7f2ff6c8 100644
--- a/src/PVE/Storage/Makefile
+++ b/src/PVE/Storage/Makefile
@@ -6,6 +6,7 @@ SOURCES= \
NFSPlugin.pm \
CIFSPlugin.pm \
ISCSIPlugin.pm \
+ NVMeOFPlugin.pm \
CephFSPlugin.pm \
RBDPlugin.pm \
ISCSIDirectPlugin.pm \
diff --git a/src/PVE/Storage/NVMeOFPlugin.pm b/src/PVE/Storage/NVMeOFPlugin.pm
new file mode 100644
index 00000000..4dc9803a
--- /dev/null
+++ b/src/PVE/Storage/NVMeOFPlugin.pm
@@ -0,0 +1,204 @@
+package PVE::Storage::NVMeOFPlugin;
+
+use strict;
+use warnings;
+
+use Cwd ();
+use Time::HiRes ();
+
+use PVE::Diskmanage;
+use PVE::Network;
+use PVE::NVMeOF;
+use PVE::Storage::Plugin;
+
+use base qw(PVE::Storage::Plugin);
+
+sub type { return 'nvmeof' }
+
+sub plugindata {
+ return {
+ content => [{ images => 1, none => 1 }, { images => 1 }],
+ select_existing => 1,
+ 'sensitive-properties' => {},
+ };
+}
+
+sub properties {
+ return {
+ subsysnqn => {
+ description => 'The NQN of the NVMe subsystem.',
+ type => 'string',
+ pattern => $PVE::NVMeOF::NQN_PATTERN,
+ maxLength => 223,
+ },
+ 'nvme-portals' => {
+ description =>
+ 'TCP portals of the NVMe subsystem, as IPv4 or bracketed IPv6 addresses'
+ . ' with an optional port (default 4420).',
+ type => 'string',
+ format => 'pve-storage-nvme-portal-list',
+ },
+ 'nvme-fast-io-fail-tmo' => {
+ description => 'Seconds before I/O through a reconnecting path fails. -1 disables'
+ . ' this timeout; I/O can remain queued while paths reconnect.',
+ type => 'integer',
+ minimum => -1,
+ default => 120,
+ },
+ };
+}
+
+sub options {
+ return {
+ subsysnqn => { fixed => 1 },
+ 'nvme-portals' => {},
+ 'nvme-fast-io-fail-tmo' => { optional => 1 },
+ nodes => { optional => 1 },
+ disable => { optional => 1 },
+ content => { optional => 1 },
+ bwlimit => { optional => 1 },
+ };
+}
+
+my sub assert_valid_config {
+ my ($scfg) = @_;
+ die "the discovery subsystem cannot be used as storage\n"
+ if $scfg->{subsysnqn} eq 'nqn.2014-08.org.nvmexpress.discovery';
+ my %seen;
+ for my $path (PVE::NVMeOF::storage_paths($scfg)->@*) {
+ my $portal = PVE::NVMeOF::format_path($path);
+ die "portal '$portal' is listed more than once\n" if $seen{$portal}++;
+ }
+}
+
+sub on_add_hook {
+ my ($class, $storeid, $scfg, %param) = @_;
+ assert_valid_config($scfg);
+ return;
+}
+
+sub on_update_hook_full {
+ my ($class, $storeid, $scfg, $update, $delete, $sensitive) = @_;
+ my $merged = { %$scfg, %$update };
+ delete $merged->{$_} for ($delete // [])->@*;
+ assert_valid_config($merged);
+ return $class->SUPER::on_update_hook_full($storeid, $scfg, $update, $delete, $sensitive);
+}
+
+sub parse_volname {
+ my ($class, $volname) = @_;
+ if ($volname =~ m!^(nvme-[a-zA-Z0-9_.:+-]+)$! && $volname !~ /-part\d+$/) {
+ return ('images', $1, undef, undef, undef, undef, 'raw');
+ }
+ die "unable to parse NVMe over Fabrics volume name '$volname'\n";
+}
+
+sub filesystem_path {
+ my ($class, $scfg, $volname, $snapname) = @_;
+ die "snapshot is not possible on NVMe over Fabrics storage\n" if defined($snapname);
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+ my $path = "/dev/disk/by-id/$name";
+ return wantarray ? ($path, $vmid, $vtype) : $path;
+}
+
+sub create_base { die "can't create base images in NVMe over Fabrics storage\n" }
+sub clone_image { die "can't clone images in NVMe over Fabrics storage\n" }
+sub alloc_image { die "can't allocate space in NVMe over Fabrics storage\n" }
+sub free_image { die "can't free space in NVMe over Fabrics storage\n" }
+sub volume_resize { die "volume resize is not possible on NVMe over Fabrics storage\n" }
+
+sub list_volumes {
+ my ($class, $storeid, $scfg, $vmid, $content_types) = @_;
+ my $res = $class->list_images($storeid, $scfg, $vmid);
+ $_->{content} = 'images' for @$res;
+ return $res;
+}
+
+sub list_images {
+ my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
+ return [] if defined($vmid) && !$vollist;
+ my $namespaces = PVE::NVMeOF::namespaces($scfg->{subsysnqn});
+ my $res = [];
+ for my $dev (sort keys %$namespaces) {
+ my $ns = $namespaces->{$dev};
+ my $volid = "$storeid:nvme-$ns->{wwid}";
+ next if $vollist && !grep { $_ eq $volid } @$vollist;
+ push @$res,
+ {
+ volid => $volid,
+ format => 'raw',
+ size => $ns->{size},
+ vmid => 0,
+ nsid => $ns->{nsid},
+ };
+ }
+ return $res;
+}
+
+sub status {
+ my ($class, $storeid, $scfg, $cache) = @_;
+ my $active = grep { $_->{state} eq 'live' }
+ values PVE::NVMeOF::subsystem_controllers($scfg->{subsysnqn})->%*;
+ return (0, 0, 0, $active ? 1 : 0);
+}
+
+sub activate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+ my $controllers = PVE::NVMeOF::subsystem_controllers($scfg->{subsysnqn});
+ for my $path (PVE::NVMeOF::storage_paths($scfg)->@*) {
+ next if defined(PVE::NVMeOF::path_controller($controllers, $path));
+ next if PVE::NVMeOF::in_cooldown($scfg, $path);
+ eval {
+ die "no answer from portal\n"
+ if !PVE::Network::tcp_ping($path->{traddr}, $path->{trsvcid}, 2);
+ PVE::NVMeOF::connect_path($scfg, $path);
+ };
+ if (my $err = $@) {
+ PVE::NVMeOF::note_connect_failure($scfg, $path);
+ warn "storage '$storeid': connecting to '"
+ . PVE::NVMeOF::format_path($path)
+ . "' failed - $err";
+ }
+ }
+ return;
+}
+
+sub deactivate_storage { return }
+
+sub check_connection {
+ my ($class, $storeid, $scfg) = @_;
+ my $controllers = PVE::NVMeOF::subsystem_controllers($scfg->{subsysnqn});
+ return 1 if grep { $_->{state} eq 'live' } values %$controllers;
+ for my $path (PVE::NVMeOF::storage_paths($scfg)->@*) {
+ next if defined(PVE::NVMeOF::path_controller($controllers, $path));
+ next if PVE::NVMeOF::in_cooldown($scfg, $path);
+ return 1 if PVE::Network::tcp_ping($path->{traddr}, $path->{trsvcid}, 2);
+ PVE::NVMeOF::note_connect_failure($scfg, $path);
+ }
+ return 0;
+}
+
+sub activate_volume {
+ my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+ my $path = $class->filesystem_path($scfg, $volname, $snapname);
+ my $deadline = Time::HiRes::time() + 10;
+ while (!-b $path && Time::HiRes::time() < $deadline) {
+ last if !keys PVE::NVMeOF::subsystem_controllers($scfg->{subsysnqn})->%*;
+ Time::HiRes::usleep(250_000);
+ }
+ my $real = Cwd::realpath($path);
+ die "volume '$volname' is not available on this node\n" if !$real || !-b $real;
+ my ($dev) = $real =~ m!^/dev/(nvme\d+n\d+)$!;
+ die "volume '$volname' is not an NVMe namespace\n" if !defined($dev);
+ my $nqn = PVE::Diskmanage::read_subsysnqn("/sys/block/$dev/device/subsysnqn") // '';
+ die "volume '$volname' is not part of subsystem '$scfg->{subsysnqn}'\n"
+ if $nqn ne $scfg->{subsysnqn};
+}
+
+sub volume_has_feature {
+ my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+ return 1 if $feature eq 'copy' && !defined($snapname);
+ return undef;
+}
+
+1;
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index e4ad7e6f..783d4145 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -42,7 +42,7 @@ our @COMMON_TAR_FLAGS = qw(
);
our @SHARED_STORAGE = (
- 'iscsi', 'nfs', 'cifs', 'rbd', 'cephfs', 'iscsidirect', 'zfs', 'drbd', 'pbs',
+ 'iscsi', 'nfs', 'cifs', 'rbd', 'cephfs', 'iscsidirect', 'zfs', 'drbd', 'pbs', 'nvmeof',
);
our $QCOW2_PREALLOCATION = {
diff --git a/src/PVE/Storage/SAN.pm b/src/PVE/Storage/SAN.pm
index bba3c128..f3682106 100644
--- a/src/PVE/Storage/SAN.pm
+++ b/src/PVE/Storage/SAN.pm
@@ -232,6 +232,11 @@ sub lun_partitions($lun) {
}
sub assert_lun_unreferenced($cfg, $lun, $exempt = undef) {
+ my $name = basename($lun->{devpath});
+ my $nqn =
+ $name =~ /^nvme\d+n\d+$/
+ ? PVE::Diskmanage::read_subsysnqn("/sys/block/$name/device/subsysnqn")
+ : undef;
my %vgs = map { $_ => 1 } PVE::Storage::LVMPlugin::lun_volume_groups(
[$lun->{devpath}, lun_partitions($lun)->@*],
)->@*;
@@ -253,6 +258,14 @@ sub assert_lun_unreferenced($cfg, $lun, $exempt = undef) {
) {
die "cannot use '$lun->{devpath}' - storage '$sid' serves images from this target\n";
}
+ if (
+ $type eq 'nvmeof'
+ && defined($nqn)
+ && $scfg->{subsysnqn} eq $nqn
+ && PVE::Storage::san_storage_serves_images($scfg)
+ ) {
+ die "cannot use '$lun->{devpath}' - storage '$sid' serves images from this subsystem\n";
+ }
next if $type ne 'lvm' || !$scfg->{base};
my (undef, $volname) = PVE::Storage::Plugin::parse_volume_id($scfg->{base});
my ($wwid) = $volname =~ m/(?:^|\.)(?:dm-uuid-mpath|scsi|nvme)-(.+)$/;
diff --git a/src/test/Makefile b/src/test/Makefile
index c9bcea8f..124612da 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -1,6 +1,6 @@
all: test
-test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint test_san_lun_scan test_san_api test_disk_api
+test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint test_san_lun_scan test_san_api test_disk_api test_nvmeof
test_zfspoolplugin: run_test_zfspoolplugin.pl
./run_test_zfspoolplugin.pl
@@ -76,3 +76,7 @@ test_san_api: run_san_api_tests.pl run_san_wipe_tests.pl run_san_nvme_tests.pl
test_disk_api: run_disk_api_tests.pl
./run_disk_api_tests.pl
+
+test_nvmeof: run_nvmeof_tests.pl run_nvmeof_taint_tests.pl
+ ./run_nvmeof_tests.pl
+ perl -T ./run_nvmeof_taint_tests.pl
diff --git a/src/test/run_nvmeof_taint_tests.pl b/src/test/run_nvmeof_taint_tests.pl
new file mode 100755
index 00000000..3c19f8cc
--- /dev/null
+++ b/src/test/run_nvmeof_taint_tests.pl
@@ -0,0 +1,60 @@
+#!/usr/bin/perl -T
+
+use v5.36;
+use lib qw(..);
+use File::Temp qw(tempdir);
+use Scalar::Util qw(tainted);
+use Test::More;
+use Test::MockModule;
+use PVE::NVMeOF;
+use PVE::Tools qw(file_set_contents file_get_contents);
+
+$ENV{PATH} = '/usr/sbin:/usr/bin:/sbin:/bin';
+delete @ENV{qw(IFS CDPATH ENV BASH_ENV)};
+my $dir = tempdir('nvmeof-taint-XXXX', DIR => '.', CLEANUP => 1);
+local $PVE::NVMeOF::HOSTNQN_FILE = "$dir/hostnqn";
+local $PVE::NVMeOF::HOSTID_FILE = "$dir/hostid";
+local $PVE::NVMeOF::RUN_DIR = "$dir/run";
+file_set_contents($PVE::NVMeOF::HOSTNQN_FILE, "nqn.host\n");
+file_set_contents($PVE::NVMeOF::HOSTID_FILE, "12345678-1234-1234-1234-123456789abc\n");
+file_set_contents("$dir/config", "nqn.test\n192.0.2.1\n120\n");
+my ($nqn, $portal, $fail) = split(/\n/, file_get_contents("$dir/config"));
+ok(tainted($nqn), 'configuration input is tainted');
+my $nvme = Test::MockModule->new('PVE::NVMeOF');
+$nvme->mock(with_lock => sub { $_[0]->() });
+my $commands = 0;
+$nvme->mock(
+ subsystem_controllers => sub {
+ return $commands ? { nvme0 => { traddr => '192.0.2.1', trsvcid => 4420 } } : {};
+ },
+);
+$nvme->mock(assert_support => sub { });
+$nvme->mock(
+ run_command => sub($cmd, %opts) {
+ $commands++;
+ is(
+ scalar(grep { tainted($_) } @$cmd),
+ 0,
+ 'all connect operands are validated and untainted',
+ );
+ PVE::Tools::run_command(['/usr/bin/true', @$cmd], timeout => 5);
+ },
+);
+my $path = PVE::NVMeOF::parse_portal($portal);
+eval { PVE::NVMeOF::connect_path({ subsysnqn => $nqn, 'nvme-fast-io-fail-tmo' => $fail }, $path) };
+is($@, '', 'connect command can execute in taint mode');
+is($commands, 1, 'one connect');
+PVE::NVMeOF::note_connect_failure({ subsysnqn => $nqn }, $path);
+ok(
+ PVE::NVMeOF::in_cooldown({ subsysnqn => $nqn }, $path),
+ 'cooldown accepts validated file-backed configuration',
+);
+for my $bad ('nqn.bad,nqn=other', "nqn.test\n", 'nqn.2014-08.org.nvmexpress.discovery') {
+ eval { PVE::NVMeOF::connect_path({ subsysnqn => $bad }, $path) };
+ like($@, qr/invalid subsystem/, 'invalid connect subsystem refused');
+}
+file_set_contents($PVE::NVMeOF::HOSTID_FILE, "00000000-0000-0000-0000-000000000000\n");
+eval { PVE::NVMeOF::host_identity() };
+like($@, qr/nonzero UUID/, 'zero host ID refused');
+
+done_testing();
diff --git a/src/test/run_nvmeof_tests.pl b/src/test/run_nvmeof_tests.pl
new file mode 100755
index 00000000..5ff96833
--- /dev/null
+++ b/src/test/run_nvmeof_tests.pl
@@ -0,0 +1,243 @@
+#!/usr/bin/perl
+
+use v5.36;
+use lib qw(..);
+use File::Temp qw(tempdir);
+use File::Path qw(make_path);
+use Fcntl qw(:flock);
+use Test::More;
+use Test::MockModule;
+use PVE::NVMeOF;
+use PVE::Storage;
+use PVE::Tools qw(file_set_contents);
+
+my $dir = tempdir('nvmeof-test-XXXX', DIR => '.', CLEANUP => 1);
+local $PVE::NVMeOF::RUN_DIR = "$dir/run";
+local $PVE::NVMeOF::SYSFS = "$dir/sys";
+my $class = 'PVE::Storage::NVMeOFPlugin';
+my $cfg = {
+ type => 'nvmeof',
+ subsysnqn => 'nqn.test',
+ 'nvme-portals' => '192.0.2.1,[2001:db8::2]:4421',
+ content => { none => 1 },
+};
+
+subtest 'configuration, portals and volumes' => sub {
+ is(PVE::Storage::Plugin->lookup('nvmeof'), $class, 'plugin registered');
+ is_deeply(
+ [sort keys $class->properties()->%*],
+ [qw(nvme-fast-io-fail-tmo nvme-portals subsysnqn)],
+ 'only three plugin-specific properties',
+ );
+ my $parsed = PVE::Storage::Plugin->parse_config(
+ 'storage.cfg',
+ "nvmeof: fabric\n\tsubsysnqn nqn.test\n\tnvme-portals 192.0.2.1\n\tcontent none\n",
+ );
+ is($parsed->{ids}->{fabric}->{subsysnqn}, 'nqn.test', 'storage.cfg accepts the plugin');
+ is_deeply(
+ $parsed->{ids}->{fabric}->{content},
+ { none => 1 },
+ 'connection-only content parses',
+ );
+ is_deeply(
+ PVE::NVMeOF::parse_portal('192.0.2.1'),
+ { traddr => '192.0.2.1', trsvcid => 4420 },
+ 'default I/O port',
+ );
+ is_deeply(
+ PVE::NVMeOF::parse_portal('[2001:0db8::1]:4421'),
+ { traddr => '2001:db8::1', trsvcid => 4421 },
+ 'canonical IPv6',
+ );
+ for my $bad (
+ 'host.example', '1.2.3.4:0', '1.2.3.4:65536', '[fe80::1]', '[::]', '224.0.0.1',
+ ) {
+ ok(!PVE::NVMeOF::parse_portal($bad), "invalid portal $bad");
+ }
+ eval {
+ $class->on_add_hook('fabric', { %$cfg, 'nvme-portals' => '192.0.2.1,192.0.2.1:4420' });
+ };
+ like($@, qr/listed more than once/, 'duplicate portals refused');
+ eval {
+ $class->on_add_hook(
+ 'fabric',
+ { %$cfg, subsysnqn => 'nqn.2014-08.org.nvmexpress.discovery' },
+ );
+ };
+ like($@, qr/discovery subsystem/, 'discovery subsystem refused');
+ is(
+ scalar($class->filesystem_path($cfg, 'nvme-eui.0123456789abcdef')),
+ '/dev/disk/by-id/nvme-eui.0123456789abcdef',
+ 'stable volume path',
+ );
+ for my $bad ('nvme-../../sda', 'nvme-eui.abc-part1', 'sda') {
+ eval { $class->parse_volname($bad) };
+ like($@, qr/unable to parse/, "invalid volume $bad");
+ }
+ my $nvme = Test::MockModule->new('PVE::NVMeOF');
+ $nvme->mock(
+ namespaces => sub { { nvme0n1 => { wwid => 'eui.abc', size => 4096, nsid => 1 } } });
+ is(
+ $class->list_volumes('fabric', $cfg)->[0]->{volid},
+ 'fabric:nvme-eui.abc',
+ 'content none still offers bases',
+ );
+ is_deeply($class->list_images('fabric', $cfg, 100), [], 'namespaces have no VM owner');
+ $nvme->mock(subsystem_controllers => sub { {} });
+ eval { $class->activate_volume('fabric', $cfg, 'nvme-nonexistent-test') };
+ like($@, qr/not available/,
+ 'missing namespace fails without waiting on absent controllers');
+};
+
+subtest 'sysfs topology' => sub {
+ my $sys = $PVE::NVMeOF::SYSFS;
+ make_path(
+ "$sys/class/nvme-fabrics/ctl/nvme1",
+ "$sys/class/nvme-subsystem/nvme-subsys0/nvme0n1",
+ "$sys/block/nvme0n1",
+ );
+ my %attrs = (
+ address => 'traddr=192.0.2.1,trsvcid=4420',
+ transport => 'tcp',
+ subsysnqn => 'nqn.test',
+ state => 'connecting',
+ );
+ file_set_contents("$sys/class/nvme-fabrics/ctl/nvme1/$_", "$attrs{$_}\n") for keys %attrs;
+ file_set_contents("$sys/class/nvme-subsystem/nvme-subsys0/subsysnqn", "nqn.test\n");
+ file_set_contents("$sys/block/nvme0n1/wwid", "eui.abc\n");
+ file_set_contents("$sys/block/nvme0n1/size", "8\n");
+ file_set_contents("$sys/block/nvme0n1/nsid", "1\n");
+ is(
+ PVE::NVMeOF::controllers()->{nvme1}->{state},
+ 'connecting',
+ 'reconnecting controller is still present',
+ );
+ is(
+ PVE::NVMeOF::path_controller(
+ PVE::NVMeOF::subsystem_controllers('nqn.test'),
+ PVE::NVMeOF::parse_portal('192.0.2.1'),
+ ),
+ 'nvme1',
+ 'portal controller found',
+ );
+ is_deeply(
+ PVE::NVMeOF::namespaces('nqn.test'),
+ { nvme0n1 => { wwid => 'eui.abc', size => 4096, nsid => 1 } },
+ 'namespace facts',
+ );
+ is_deeply(PVE::NVMeOF::namespaces('nqn.other'), {}, 'other subsystem excluded');
+};
+
+subtest 'serialized commands inherit the lock' => sub {
+ my $value = PVE::NVMeOF::with_lock(sub {
+ my $out = '';
+ PVE::Tools::run_command(
+ [
+ '/usr/bin/perl',
+ '-MFcntl=:flock',
+ '-e',
+ 'my @held = grep { (readlink($_) // "") =~ /fabrics.lock$/ } glob("/proc/self/fd/*"); die "not inherited" if !@held; open(my $fh, ">>", $ARGV[0]) or die $!; die "lock missing" if flock($fh, LOCK_EX|LOCK_NB); print "locked\\n";',
+ "$PVE::NVMeOF::RUN_DIR/fabrics.lock",
+ ],
+ timeout => 5,
+ outfunc => sub { $out .= $_[0] },
+ );
+ return $out;
+ });
+ is($value, 'locked', 'exec cannot acquire the lock held by its parent');
+ open(my $lock, '>>', "$PVE::NVMeOF::RUN_DIR/fabrics.lock") or die $!;
+ flock($lock, LOCK_EX) or die $!;
+ eval {
+ PVE::NVMeOF::with_lock(sub { die "must not count paths\n" });
+ };
+ like($@, qr/is busy/, 'busy operation never enters callback');
+ close($lock);
+
+ my $tools = Test::MockModule->new('PVE::Tools');
+ my $timed_out = 1;
+ $tools->mock(run_fork_with_timeout => sub { return ($_[1]->(), $timed_out) });
+ my $retained;
+ eval {
+ PVE::NVMeOF::with_lock(sub {
+ my ($path) =
+ grep { (readlink($_) // '') =~ /fabrics.lock$/ } glob('/proc/self/fd/*');
+ my ($fd) = $path =~ m{/(\d+)$};
+ open($retained, '>&', $fd) or die $!;
+ return 1;
+ });
+ };
+ like($@, qr/timed out.*may still be running/, 'timeout does not claim deletion completed');
+ $timed_out = 0;
+ eval {
+ PVE::NVMeOF::with_lock(sub { die "must not count paths\n" });
+ };
+ like($@, qr/is busy/, 'retained descriptor keeps serialization after the caller returns');
+ close($retained);
+ is(PVE::NVMeOF::with_lock(sub { 1 }), 1, 'last descriptor close releases serialization');
+};
+
+subtest 'connect options, kernel reconnect and cooldown' => sub {
+ my $nvme = Test::MockModule->new('PVE::NVMeOF');
+ my $net = Test::MockModule->new('PVE::Network');
+ my @commands;
+ my $controllers = {};
+ $nvme->mock(with_lock => sub { $_[0]->() });
+ $nvme->mock(assert_support => sub { });
+ $nvme->mock(host_identity =>
+ sub { { nqn => 'nqn.host', id => '12345678-1234-1234-1234-123456789abc' } });
+ $nvme->mock(subsystem_controllers => sub { $controllers });
+ my $appear = 1;
+ $nvme->mock(
+ run_command => sub($cmd, %opts) {
+ is($opts{timeout}, 30, 'bounded connect');
+ push @commands, $cmd;
+ my %arg =
+ map { $cmd->[$_] => $cmd->[$_ + 1] } grep { $cmd->[$_] =~ /^--tr/ } 0 .. $#$cmd;
+ $controllers->{ 'nvme' . scalar(@commands) } =
+ { traddr => $arg{'--traddr'}, trsvcid => $arg{'--trsvcid'}, state => 'live' }
+ if $appear;
+ },
+ );
+ $net->mock(tcp_ping => sub { 1 });
+ $class->activate_storage('fabric', $cfg);
+ is(scalar(@commands), 2, 'one connect per missing portal');
+ ok(grep({ $_ eq '--ctrl-loss-tmo=-1' } $commands[0]->@*), 'kernel reconnects indefinitely');
+ is_deeply([@{ $commands[0] }[-2, -1]], ['--fast_io_fail_tmo', 120], 'fast failure default');
+ @commands = ();
+ $controllers = {
+ nvme1 => { traddr => '192.0.2.1', trsvcid => 4420, state => 'connecting' },
+ nvme2 => { traddr => '2001:db8::2', trsvcid => 4421, state => 'live' },
+ };
+ $class->activate_storage('fabric', $cfg);
+ is_deeply(\@commands, [],
+ 'existing controllers are never reconnected, regardless of state');
+ $class->deactivate_storage('fabric', $cfg);
+ is_deeply(\@commands, [], 'deactivation does not remove controllers');
+ $controllers = {};
+ my $path = PVE::NVMeOF::storage_paths($cfg)->[0];
+ PVE::NVMeOF::note_connect_failure($cfg, $path);
+ ok(PVE::NVMeOF::in_cooldown($cfg, $path), 'failure starts cooldown');
+ $class->activate_storage('fabric', $cfg);
+ is(scalar(@commands), 1, 'only the other portal connects');
+ ($controllers, $appear) = ({}, 0);
+ my $other = PVE::NVMeOF::storage_paths($cfg)->[1];
+ my $warning = '';
+ {
+ local $SIG{__WARN__} = sub { $warning .= $_[0] };
+ $class->activate_storage('fabric', $cfg);
+ }
+ like($warning, qr/no controller appeared/, 'a connect without a controller is a failure');
+ ok(PVE::NVMeOF::in_cooldown($cfg, $other), 'which starts the cooldown too');
+};
+
+subtest 'local disk setup rejects fabric namespaces by transport' => sub {
+ my $disk = Test::MockModule->new('PVE::Diskmanage');
+ for my $transport (qw(nvme-tcp nvme-rdma nvme-fc)) {
+ $disk->mock(get_disks => sub { { nvme0n1 => { transport => $transport } } });
+ eval { PVE::Diskmanage::assert_disk_unused('/dev/nvme0n1') };
+ like($@, qr/not a valid local disk/,
+ "$transport refuses without storage configuration");
+ }
+};
+
+done_testing();
diff --git a/src/test/run_san_nvme_tests.pl b/src/test/run_san_nvme_tests.pl
index c2f3c65e..b1a39629 100755
--- a/src/test/run_san_nvme_tests.pl
+++ b/src/test/run_san_nvme_tests.pl
@@ -38,6 +38,7 @@ $storage->mock(config => sub { $cfg });
$disk->mock(verify_blockdev_path => sub { $device });
$disk->mock(is_partition => sub { 0 });
$disk->mock(get_nvme_transport => sub { 'fc' });
+$disk->mock(read_subsysnqn => sub { 'nqn.test' });
$disk->mock(is_iscsi_device => sub { 0 });
$disk->mock(is_mounted => sub { 0 });
$disk->mock(has_holder => sub { 0 });
@@ -65,6 +66,18 @@ $disk->mock(is_partition => sub { 0 });
$cfg->{ids}->{data} = { type => 'lvm', base => 'fabric:nvme-eui.0123456789abcdef' };
eval { PVE::Storage::SAN::assert_lun_unreferenced($cfg, $lun) };
like($@, qr/base volume/, 'native base reference refuses');
+$cfg->{ids} =
+ { fabric =>
+ { type => 'nvmeof', subsysnqn => 'nqn.test', disable => 1, content => { images => 1 } } };
+eval { PVE::Storage::SAN::assert_lun_unreferenced($cfg, $lun) };
+like(
+ $@,
+ qr/serves images from this subsystem/,
+ 'direct namespace image use refuses even when disabled',
+);
+$cfg->{ids}->{fabric}->{content} = { none => 1 };
+eval { PVE::Storage::SAN::assert_lun_unreferenced($cfg, $lun) };
+is($@, '', 'connection-only storage permits namespace initialization');
$cfg->{ids} = {};
@order = @opens = ();
$tools->mock(file_get_contents => sub { 1024 });
next prev parent reply other threads:[~2026-11-13 3:37 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-13 3:04 [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 01/17] diskmanage: skip SMART queries for NVMe fabric namespaces Thomas Lamprecht
2026-11-13 3:04 ` Thomas Lamprecht [this message]
2026-11-13 3:04 ` [PATCH storage 03/17] lvm: allow a volume group on an NVMe over Fabrics namespace Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 04/17] api: scan: add NVMe over Fabrics discovery Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 05/17] pvesm: add a scan command for NVMe over Fabrics Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 06/17] api: scan: san-luns: attribute NVMe namespaces to storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 07/17] api: add node-level NVMe over Fabrics status and controller removal Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 08/17] multipath: report the NVMe host NQN of each node Thomas Lamprecht
2026-11-13 3:04 ` [PATCH storage 09/17] multipath: report NVMe over Fabrics namespaces in the cluster status Thomas Lamprecht
2026-11-13 3:04 ` [PATCH docs 10/17] storage: document NVMe over Fabrics storages Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 11/17] api: nodes: mount the NVMe over Fabrics endpoints Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 12/17] ui: storage: add NVMe over Fabrics storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 13/17] ui: lvm: offer NVMe over Fabrics storages as base storage Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 14/17] ui: dc: san: add a guided setup for shared LVM on NVMe over Fabrics Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 15/17] ui: node: san: list NVMe over Fabrics controllers Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 16/17] ui: dc: san: show the NVMe host NQN of each node Thomas Lamprecht
2026-11-13 3:04 ` [PATCH manager 17/17] ui: dc: san: show that a namespace uses native multipath Thomas Lamprecht
2026-11-18 20:44 ` superseded: [PATCH storage,docs,manager 00/17] NVMe over Fabrics storage Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261113030541.1591528-3-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.