* [PATCH storage v2 2/7] zfs: add native NVMe/TCP storage backend
2026-08-02 3:31 [PATCH storage v2 0/7] add native ZFS over NVMe/TCP backend Joaquin Varela
2026-08-02 3:31 ` [PATCH storage v2 1/7] zfs: make LUN provider dispatch overridable Joaquin Varela
@ 2026-08-02 3:31 ` Joaquin Varela
2026-08-02 3:31 ` [PATCH storage v2 3/7] zfsnvme: harden node preflight and storage teardown Joaquin Varela
` (4 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Joaquin Varela @ 2026-08-02 3:31 UTC (permalink / raw)
To: pve-devel; +Cc: Joaquin Varela
Add the zfsnvme shared-storage type and an NVMET LUN provider.
ZFS continues to own volume lifecycle. Linux nvmet publishes
each zvol as a namespace, and native NVMe multipath combines
the configured paths.
Persist subsystem NQN, NSID and UUID as ZFS user properties.
Configfs is derived state and reconcile can rebuild it after a
target restart. Use the namespace UUID for the stable device
path exposed to QEMU.
Bind each connection to its configured host interface, require
native multipath, and support DH-HMAC-CHAP without exposing
secrets in argv. Add tests for parsing, identity, lifecycle and
connect behavior.
Signed-off-by: Joaquin Varela <joaquinvarela@neatech.ar>
---
debian/control | 1 +
src/PVE/Storage.pm | 2 +
src/PVE/Storage/LunCmd/Makefile | 2 +-
src/PVE/Storage/LunCmd/NVMET.pm | 693 ++++++++++++++++++++++++++++++
src/PVE/Storage/Makefile | 1 +
src/PVE/Storage/Plugin.pm | 2 +-
src/PVE/Storage/ZFSNVMePlugin.pm | 716 +++++++++++++++++++++++++++++++
src/test/run_plugin_tests.pl | 1 +
src/test/zfsnvme_test.pm | 350 +++++++++++++++
9 files changed, 1766 insertions(+), 2 deletions(-)
create mode 100644 src/PVE/Storage/LunCmd/NVMET.pm
create mode 100644 src/PVE/Storage/ZFSNVMePlugin.pm
create mode 100644 src/test/zfsnvme_test.pm
diff --git a/debian/control b/debian/control
index 3ad25e7..ac770a4 100644
--- a/debian/control
+++ b/debian/control
@@ -43,6 +43,7 @@ Depends: bzip2,
lvm2,
lzop,
nfs-common,
+ nvme-cli,
proxmox-backup-client (>= 2.1.10~),
proxmox-backup-file-restore,
pve-cluster (>= 5.0-32),
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 64ea9da..4356d83 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -36,6 +36,7 @@ use PVE::Storage::CephFSPlugin;
use PVE::Storage::ISCSIDirectPlugin;
use PVE::Storage::ZFSPoolPlugin;
use PVE::Storage::ZFSPlugin;
+use PVE::Storage::ZFSNVMePlugin;
use PVE::Storage::PBSPlugin;
use PVE::Storage::BTRFSPlugin;
use PVE::Storage::ESXiPlugin;
@@ -61,6 +62,7 @@ PVE::Storage::CephFSPlugin->register();
PVE::Storage::ISCSIDirectPlugin->register();
PVE::Storage::ZFSPoolPlugin->register();
PVE::Storage::ZFSPlugin->register();
+PVE::Storage::ZFSNVMePlugin->register();
PVE::Storage::PBSPlugin->register();
PVE::Storage::BTRFSPlugin->register();
PVE::Storage::ESXiPlugin->register();
diff --git a/src/PVE/Storage/LunCmd/Makefile b/src/PVE/Storage/LunCmd/Makefile
index a7209d1..2d30cad 100644
--- a/src/PVE/Storage/LunCmd/Makefile
+++ b/src/PVE/Storage/LunCmd/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Comstar.pm Istgt.pm Iet.pm LIO.pm
+SOURCES=Comstar.pm Istgt.pm Iet.pm LIO.pm NVMET.pm
.PHONY: install
install:
diff --git a/src/PVE/Storage/LunCmd/NVMET.pm b/src/PVE/Storage/LunCmd/NVMET.pm
new file mode 100644
index 0000000..7a902da
--- /dev/null
+++ b/src/PVE/Storage/LunCmd/NVMET.pm
@@ -0,0 +1,693 @@
+package PVE::Storage::LunCmd::NVMET;
+
+use v5.36;
+
+use PVE::Tools qw(run_command trim);
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+my $id_rsa_path = '/etc/pve/priv/zfs';
+
+my $REMOTE_HELPER = <<'REMOTE_HELPER';
+set -euo pipefail
+
+ROOT=/sys/kernel/config/nvmet
+MODEL='Proxmox ZFS NVMe'
+PROP_SUBSYS='proxmox:nvme-subsys'
+PROP_NSID='proxmox:nvme-nsid'
+PROP_UUID='proxmox:nvme-uuid'
+
+die() {
+ echo "$*" >&2
+ exit 1
+}
+
+validate_nqn() {
+ local value="$1"
+ [[ "$value" =~ ^nqn\.[A-Za-z0-9][A-Za-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._:-]*$ ]] ||
+ die "invalid NQN"
+ ((${#value} <= 223)) || die "NQN is too long"
+}
+
+validate_pool() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.:/-]*$ ]] || die "invalid ZFS pool name"
+}
+
+validate_device() {
+ [[ "$1" =~ ^/dev/zvol/[A-Za-z0-9][A-Za-z0-9_.:/-]*$ ]] ||
+ die "invalid zvol device path"
+}
+
+validate_uuid() {
+ [[ "$1" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] ||
+ die "invalid namespace UUID"
+}
+
+prepare_configfs() {
+ modprobe nvmet_tcp
+ if ! mountpoint -q /sys/kernel/config; then
+ mount -t configfs none /sys/kernel/config
+ fi
+ [[ -d "$ROOT/subsystems" && -d "$ROOT/ports" && -d "$ROOT/hosts" ]] ||
+ die "NVMe target configfs is unavailable"
+
+ install -d -m 0755 /run/lock
+ exec 9>/run/lock/pve-nvmet.lock
+ flock -w 30 9 || die "timed out waiting for NVMe target configuration lock"
+}
+
+ensure_subsystem() {
+ local nqn="$1"
+ local subsystem="$ROOT/subsystems/$nqn"
+ local serial
+
+ validate_nqn "$nqn"
+ serial="PVEZFS$(printf '%s' "$nqn" | sha256sum | cut -c1-14)"
+
+ if [[ ! -d "$subsystem" ]]; then
+ mkdir "$subsystem"
+ printf '%s\n' "$MODEL" >"$subsystem/attr_model"
+ printf '%s\n' "$serial" >"$subsystem/attr_serial"
+ else
+ local current_model current_serial
+ current_model="$(<"$subsystem/attr_model")"
+ current_serial="$(<"$subsystem/attr_serial")"
+ if [[ "$current_model" != "$MODEL" || "$current_serial" != "$serial" ]]; then
+ if find "$subsystem/namespaces" -mindepth 1 -maxdepth 1 -type d -print -quit |
+ grep -q .; then
+ die "refusing to take over existing NVMe subsystem '$nqn'"
+ fi
+ printf '%s\n' "$MODEL" >"$subsystem/attr_model"
+ printf '%s\n' "$serial" >"$subsystem/attr_serial"
+ fi
+ fi
+
+ printf '0\n' >"$subsystem/attr_allow_any_host"
+}
+
+find_port() {
+ local family="$1"
+ local address="$2"
+ local service="$3"
+ local port
+
+ for port in "$ROOT"/ports/*; do
+ [[ -d "$port" ]] || continue
+ [[ "$(<"$port/addr_trtype")" == tcp ]] || continue
+ [[ "$(<"$port/addr_adrfam")" == "$family" ]] || continue
+ [[ "$(<"$port/addr_traddr")" == "$address" ]] || continue
+ [[ "$(<"$port/addr_trsvcid")" == "$service" ]] || continue
+ basename "$port"
+ return 0
+ done
+ return 1
+}
+
+allocate_port_id() {
+ local id=1
+ while [[ -e "$ROOT/ports/$id" ]]; do
+ ((id++))
+ done
+ printf '%s\n' "$id"
+}
+
+ensure_port() {
+ local nqn="$1"
+ local spec="$2"
+ local family address service id port
+
+ IFS=, read -r family address service <<<"$spec"
+ [[ "$family" == ipv4 || "$family" == ipv6 ]] || die "invalid address family"
+ [[ "$service" =~ ^[0-9]{1,5}$ ]] || die "invalid NVMe/TCP service"
+ ((service >= 1 && service <= 65535)) || die "invalid NVMe/TCP service"
+ if [[ "$family" == ipv4 ]]; then
+ [[ "$address" =~ ^[0-9.]+$ ]] || die "invalid IPv4 target address"
+ else
+ [[ "$address" =~ ^[0-9a-fA-F:]+$ ]] || die "invalid IPv6 target address"
+ fi
+
+ id="$(find_port "$family" "$address" "$service" || true)"
+ if [[ -z "$id" ]]; then
+ id="$(allocate_port_id)"
+ port="$ROOT/ports/$id"
+ mkdir "$port"
+ printf 'tcp\n' >"$port/addr_trtype"
+ printf '%s\n' "$family" >"$port/addr_adrfam"
+ printf '%s\n' "$address" >"$port/addr_traddr"
+ printf '%s\n' "$service" >"$port/addr_trsvcid"
+ fi
+
+ port="$ROOT/ports/$id"
+ if [[ ! -e "$port/subsystems/$nqn" ]]; then
+ ln -s "$ROOT/subsystems/$nqn" "$port/subsystems/$nqn"
+ fi
+ printf '%s\n' "$id"
+}
+
+ensure_target() {
+ local nqn="$1"
+ local pool="$2"
+ shift 2
+ local spec id port desired_ports=''
+
+ validate_nqn "$nqn"
+ validate_pool "$pool"
+ (($# >= 1)) || die "at least one NVMe/TCP portal is required"
+ ensure_subsystem "$nqn"
+
+ for spec in "$@"; do
+ id="$(ensure_port "$nqn" "$spec")"
+ desired_ports="$desired_ports $id"
+ done
+
+ for port in "$ROOT"/ports/*; do
+ [[ -d "$port" && -L "$port/subsystems/$nqn" ]] || continue
+ id="$(basename "$port")"
+ case " $desired_ports " in
+ *" $id "*) ;;
+ *) rm "$port/subsystems/$nqn" ;;
+ esac
+ done
+}
+
+ensure_host() {
+ local nqn="$1"
+ local hostnqn="$2"
+ validate_nqn "$nqn"
+ validate_nqn "$hostnqn"
+ [[ -d "$ROOT/subsystems/$nqn" ]] || die "NVMe subsystem does not exist"
+ [[ -d "$ROOT/hosts/$hostnqn" ]] || mkdir "$ROOT/hosts/$hostnqn"
+}
+
+allow_host() {
+ local nqn="$1"
+ local hostnqn="$2"
+ validate_nqn "$nqn"
+ validate_nqn "$hostnqn"
+ [[ -d "$ROOT/hosts/$hostnqn" ]] || die "NVMe host does not exist"
+ if [[ ! -e "$ROOT/subsystems/$nqn/allowed_hosts/$hostnqn" ]]; then
+ ln -s "$ROOT/hosts/$hostnqn" "$ROOT/subsystems/$nqn/allowed_hosts/$hostnqn"
+ fi
+}
+
+list_volumes() {
+ local pool="$1"
+ local dataset property value source
+ declare -A datasets=()
+ declare -A subsystems=()
+ declare -A nsids=()
+ declare -A uuids=()
+
+ while IFS=$'\t' read -r dataset property value source; do
+ datasets["$dataset"]=1
+ if [[ "$source" != local && "$source" != received ]]; then
+ value='-'
+ fi
+ case "$property" in
+ "$PROP_SUBSYS") subsystems["$dataset"]="$value" ;;
+ "$PROP_NSID") nsids["$dataset"]="$value" ;;
+ "$PROP_UUID") uuids["$dataset"]="$value" ;;
+ esac
+ done < <(
+ zfs get -H -r -t volume -o name,property,value,source \
+ "$PROP_SUBSYS,$PROP_NSID,$PROP_UUID" "$pool"
+ )
+
+ for dataset in "${!datasets[@]}"; do
+ printf '%s\t%s\t%s\t%s\n' \
+ "$dataset" \
+ "${subsystems[$dataset]:--}" \
+ "${nsids[$dataset]:--}" \
+ "${uuids[$dataset]:--}"
+ done
+}
+
+property_value() {
+ local property="$1"
+ local dataset="$2"
+ local output value source
+ output="$(zfs get -H -o value,source "$property" "$dataset" 2>/dev/null || true)"
+ IFS=$'\t' read -r value source <<<"$output"
+ if [[ -z "$value" || ("$source" != local && "$source" != received) ]]; then
+ value='-'
+ fi
+ printf '%s\n' "$value"
+}
+
+identity_is_duplicate() {
+ local pool="$1"
+ local dataset="$2"
+ local nqn="$3"
+ local nsid="$4"
+ local uuid="$5"
+ local other other_nqn other_nsid other_uuid
+
+ while IFS=$'\t' read -r other other_nqn other_nsid other_uuid; do
+ [[ "$other" != "$dataset" && "$other_nqn" == "$nqn" ]] || continue
+ [[ "$other_nsid" == "$nsid" || "$other_uuid" == "$uuid" ]] && return 0
+ done < <(list_volumes "$pool")
+ return 1
+}
+
+allocate_nsid() {
+ local pool="$1"
+ local nqn="$2"
+ local dataset other_nqn other_nsid other_uuid namespace nsid=1
+ declare -A used=()
+
+ while IFS=$'\t' read -r dataset other_nqn other_nsid other_uuid; do
+ if [[ "$other_nqn" == "$nqn" && "$other_nsid" =~ ^[1-9][0-9]*$ ]]; then
+ used["$other_nsid"]=1
+ fi
+ done < <(list_volumes "$pool")
+
+ for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+ [[ -d "$namespace" ]] || continue
+ used["$(basename "$namespace")"]=1
+ done
+
+ while [[ -n "${used[$nsid]:-}" ]]; do
+ ((nsid++))
+ done
+ printf '%s\n' "$nsid"
+}
+
+ensure_identity() {
+ local nqn="$1"
+ local pool="$2"
+ local dataset="$3"
+ local current_nqn nsid uuid
+
+ current_nqn="$(property_value "$PROP_SUBSYS" "$dataset")"
+ nsid="$(property_value "$PROP_NSID" "$dataset")"
+ uuid="$(property_value "$PROP_UUID" "$dataset")"
+
+ if [[ "$current_nqn" != '-' && "$current_nqn" != "$nqn" ]]; then
+ die "ZFS volume '$dataset' belongs to NVMe subsystem '$current_nqn'"
+ fi
+
+ if [[ ! "$nsid" =~ ^[1-9][0-9]*$ ]] ||
+ [[ ! "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] ||
+ identity_is_duplicate "$pool" "$dataset" "$nqn" "$nsid" "$uuid"; then
+ nsid="$(allocate_nsid "$pool" "$nqn")"
+ uuid="$(< /proc/sys/kernel/random/uuid)"
+ fi
+
+ validate_uuid "$uuid"
+ zfs set "$PROP_SUBSYS=$nqn" "$PROP_NSID=$nsid" "$PROP_UUID=$uuid" "$dataset"
+ IDENTITY_NSID="$nsid"
+ IDENTITY_UUID="$uuid"
+}
+
+remove_namespace() {
+ local namespace="$1"
+ if [[ "$(<"$namespace/enable")" == 1 ]]; then
+ printf '0\n' >"$namespace/enable"
+ fi
+ rmdir "$namespace"
+}
+
+ensure_namespace() {
+ local nqn="$1"
+ local nsid="$2"
+ local uuid="$3"
+ local device="$4"
+ local namespace="$ROOT/subsystems/$nqn/namespaces/$nsid"
+ local other
+
+ validate_uuid "$uuid"
+ validate_device "$device"
+ [[ -b "$device" ]] || die "zvol '$device' is not a block device"
+ [[ -d "$ROOT/subsystems/$nqn" ]] || die "NVMe subsystem does not exist"
+
+ for other in "$ROOT/subsystems/$nqn"/namespaces/*; do
+ [[ -d "$other" && "$other" != "$namespace" ]] || continue
+ if [[ "$(<"$other/device_uuid")" == "$uuid" ]]; then
+ die "namespace UUID '$uuid' is already in use"
+ fi
+ done
+
+ if [[ -d "$namespace" ]]; then
+ if [[ "$(<"$namespace/device_uuid")" == "$uuid" ]] &&
+ [[ "$(<"$namespace/device_path")" == "$device" ]]; then
+ [[ "$(<"$namespace/enable")" == 1 ]] || printf '1\n' >"$namespace/enable"
+ return
+ fi
+ remove_namespace "$namespace"
+ fi
+
+ mkdir "$namespace"
+ if ! {
+ printf '%s\n' "$device" >"$namespace/device_path"
+ printf '%s\n' "$uuid" >"$namespace/device_uuid"
+ printf '0\n' >"$namespace/buffered_io"
+ printf '1\n' >"$namespace/enable"
+ }; then
+ printf '0\n' >"$namespace/enable" 2>/dev/null || true
+ rmdir "$namespace" 2>/dev/null || true
+ die "failed to create namespace '$nsid'"
+ fi
+}
+
+create_volume() {
+ local nqn="$1"
+ local pool="$2"
+ local device="$3"
+ local dataset
+
+ validate_nqn "$nqn"
+ validate_pool "$pool"
+ validate_device "$device"
+ dataset="${device#/dev/zvol/}"
+ [[ "$dataset" == "$pool/"* ]] || die "zvol is outside configured pool"
+ [[ "$(zfs get -H -o value type "$dataset")" == volume ]] ||
+ die "'$dataset' is not a ZFS volume"
+
+ ensure_identity "$nqn" "$pool" "$dataset"
+ ensure_namespace "$nqn" "$IDENTITY_NSID" "$IDENTITY_UUID" "$device"
+ printf '%s\n' "$IDENTITY_UUID"
+}
+
+delete_volume() {
+ local nqn="$1"
+ local uuid="$2"
+ local namespace found=0
+
+ validate_nqn "$nqn"
+ validate_uuid "$uuid"
+ [[ -d "$ROOT/subsystems/$nqn" ]] || return 0
+ for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+ [[ -d "$namespace" ]] || continue
+ [[ "$(<"$namespace/device_uuid")" == "$uuid" ]] || continue
+ ((found == 0)) || die "duplicate namespace UUID '$uuid'"
+ remove_namespace "$namespace"
+ found=1
+ done
+}
+
+lookup_volume() {
+ local nqn="$1"
+ local pool="$2"
+ local device="$3"
+ local dataset current_nqn nsid uuid
+
+ validate_nqn "$nqn"
+ validate_pool "$pool"
+ validate_device "$device"
+ dataset="${device#/dev/zvol/}"
+ [[ "$dataset" == "$pool/"* ]] || die "zvol is outside configured pool"
+ [[ "$(zfs get -H -o value type "$dataset")" == volume ]] ||
+ die "'$dataset' is not a ZFS volume"
+
+ current_nqn="$(property_value "$PROP_SUBSYS" "$dataset")"
+ nsid="$(property_value "$PROP_NSID" "$dataset")"
+ uuid="$(property_value "$PROP_UUID" "$dataset")"
+ [[ "$current_nqn" == "$nqn" ]] ||
+ die "ZFS volume '$dataset' is not owned by NVMe subsystem '$nqn'"
+ [[ "$nsid" =~ ^[1-9][0-9]*$ ]] || die "invalid NSID on '$dataset'"
+ validate_uuid "$uuid"
+ identity_is_duplicate "$pool" "$dataset" "$nqn" "$nsid" "$uuid" &&
+ die "duplicate NVMe identity on '$dataset'"
+
+ ensure_namespace "$nqn" "$nsid" "$uuid" "$device"
+ printf '%s\n' "$uuid"
+}
+
+lookup_view() {
+ local nqn="$1"
+ local pool="$2"
+ local uuid="$3"
+ local dataset other_nqn nsid other_uuid found=''
+
+ validate_nqn "$nqn"
+ validate_pool "$pool"
+ validate_uuid "$uuid"
+ while IFS=$'\t' read -r dataset other_nqn nsid other_uuid; do
+ [[ "$other_nqn" == "$nqn" && "$other_uuid" == "$uuid" ]] || continue
+ [[ -z "$found" ]] || die "duplicate namespace UUID '$uuid'"
+ found="$nsid"
+ done < <(list_volumes "$pool")
+ [[ "$found" =~ ^[1-9][0-9]*$ ]] || die "namespace UUID '$uuid' was not found"
+ printf '%s\n' "$found"
+}
+
+resize_volume() {
+ local nqn="$1"
+ local uuid="$2"
+ local namespace
+
+ validate_nqn "$nqn"
+ validate_uuid "$uuid"
+ for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+ [[ -d "$namespace" ]] || continue
+ if [[ "$(<"$namespace/device_uuid")" == "$uuid" ]]; then
+ printf '1\n' >"$namespace/revalidate_size"
+ return
+ fi
+ done
+ die "namespace UUID '$uuid' was not found"
+}
+
+reconcile() {
+ local nqn="$1"
+ local pool="$2"
+ local dataset other_nqn nsid uuid device namespace
+ declare -A desired=()
+ declare -A seen_uuid=()
+
+ validate_nqn "$nqn"
+ validate_pool "$pool"
+ [[ -d "$ROOT/subsystems/$nqn" ]] || die "NVMe subsystem does not exist"
+
+ while IFS=$'\t' read -r dataset other_nqn nsid uuid; do
+ [[ "$other_nqn" == "$nqn" ]] || continue
+ [[ "$nsid" =~ ^[1-9][0-9]*$ ]] || die "invalid NSID on '$dataset'"
+ validate_uuid "$uuid"
+ [[ -z "${desired[$nsid]:-}" ]] || die "duplicate NSID '$nsid'"
+ [[ -z "${seen_uuid[$uuid]:-}" ]] || die "duplicate namespace UUID '$uuid'"
+ desired["$nsid"]=1
+ seen_uuid["$uuid"]=1
+ device="/dev/zvol/$dataset"
+ ensure_namespace "$nqn" "$nsid" "$uuid" "$device"
+ done < <(list_volumes "$pool")
+
+ for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+ [[ -d "$namespace" ]] || continue
+ nsid="$(basename "$namespace")"
+ [[ -n "${desired[$nsid]:-}" ]] || remove_namespace "$namespace"
+ done
+}
+
+delete_target() {
+ local nqn="$1"
+ local subsystem="$ROOT/subsystems/$nqn"
+ local namespace port link hostnqn referenced
+ local -a hosts=()
+
+ validate_nqn "$nqn"
+ [[ -d "$subsystem" ]] || return 0
+
+ for namespace in "$subsystem"/namespaces/*; do
+ [[ -d "$namespace" ]] || continue
+ remove_namespace "$namespace"
+ done
+ for port in "$ROOT"/ports/*; do
+ [[ -d "$port" ]] || continue
+ link="$port/subsystems/$nqn"
+ [[ -L "$link" ]] && rm "$link"
+ done
+ for link in "$subsystem"/allowed_hosts/*; do
+ [[ -L "$link" ]] || continue
+ hosts+=("$(basename "$link")")
+ rm "$link"
+ done
+ rmdir "$subsystem"
+
+ # Host objects (and their DHCHAP keys) are global in nvmet. Remove an
+ # orphan only after proving that no other subsystem still authorizes it.
+ for hostnqn in "${hosts[@]}"; do
+ referenced=0
+ for link in "$ROOT"/subsystems/*/allowed_hosts/"$hostnqn"; do
+ if [[ -L "$link" ]]; then
+ referenced=1
+ break
+ fi
+ done
+ ((referenced == 1)) || rmdir "$ROOT/hosts/$hostnqn"
+ done
+}
+
+prepare_configfs
+mode="${1:-}"
+shift || true
+
+case "$mode" in
+ ensure-target) ensure_target "$@" ;;
+ ensure-host) ensure_host "$@" ;;
+ allow-host) allow_host "$@" ;;
+ create) create_volume "$@" ;;
+ delete) delete_volume "$@" ;;
+ lookup) lookup_volume "$@" ;;
+ view) lookup_view "$@" ;;
+ resize) resize_volume "$@" ;;
+ reconcile) reconcile "$@" ;;
+ delete-target) delete_target "$@" ;;
+ *) die "unknown NVMe target operation" ;;
+esac
+REMOTE_HELPER
+
+my $REMOTE_SET_HOST_KEY = <<'REMOTE_SET_HOST_KEY';
+set -euo pipefail
+
+ROOT=/sys/kernel/config/nvmet
+hostnqn="$1"
+host="$ROOT/hosts/$hostnqn"
+
+[[ "$hostnqn" =~ ^nqn\.[A-Za-z0-9][A-Za-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._:-]*$ ]] || {
+ echo "invalid host NQN '$hostnqn'" >&2
+ exit 1
+}
+((${#hostnqn} <= 223)) || {
+ echo 'host NQN is too long' >&2
+ exit 1
+}
+[[ -d "$host" ]] || {
+ echo 'NVMe host does not exist' >&2
+ exit 1
+}
+
+IFS= read -r key
+[[ "$key" =~ ^DHHC-1:[0-9A-Fa-f]{2}:[A-Za-z0-9+/=]+:$ ]] || {
+ echo 'invalid DH-HMAC-CHAP key representation' >&2
+ exit 1
+}
+current="$(<"$host/dhchap_key")"
+if [[ -n "$current" && "$current" != "$key" ]]; then
+ for link in "$ROOT"/subsystems/*/allowed_hosts/"$hostnqn"; do
+ if [[ -L "$link" ]]; then
+ echo "refusing to replace an in-use DH-HMAC-CHAP key for '$hostnqn'" >&2
+ exit 1
+ fi
+ done
+fi
+printf '%s\n' "$key" >"$host/dhchap_key"
+REMOTE_SET_HOST_KEY
+
+my sub server($scfg) {
+ return $scfg->{server} // $scfg->{portal};
+}
+
+my sub ssh_key($scfg) {
+ my $server = server($scfg);
+ return "$id_rsa_path/${server}_id_rsa";
+}
+
+my sub remote_call($scfg, $timeout, $operation, @params) {
+ my $server = server($scfg);
+ my $target = 'root@' . $server;
+ my $msg = '';
+ my $err = '';
+ my $cmd = [
+ @ssh_cmd,
+ '-i',
+ ssh_key($scfg),
+ $target,
+ '--',
+ '/bin/bash',
+ '-s',
+ '--',
+ $operation,
+ @params,
+ ];
+
+ run_command(
+ $cmd,
+ input => $REMOTE_HELPER,
+ timeout => $timeout // 10,
+ outfunc => sub($line) { $msg .= "$line\n" },
+ errfunc => sub($line) { $err .= "$line\n" },
+ errmsg => "NVMe target operation '$operation' failed",
+ );
+
+ return trim($msg);
+}
+
+sub get_base($scfg) {
+ return '/dev/zvol';
+}
+
+sub ensure_target($scfg, $hostnqn, $portals) {
+ my $nqn = $scfg->{subsysnqn};
+
+ remote_call($scfg, 15, 'ensure-target', $nqn, $scfg->{pool}, $portals->@*);
+ remote_call($scfg, 10, 'ensure-host', $nqn, $hostnqn);
+}
+
+sub set_host_key($scfg, $hostnqn, $key) {
+ my $server = server($scfg);
+ my $target = 'root@' . $server;
+ my $remote_cmd = join(
+ ' ',
+ map { PVE::Tools::shellquote($_) }
+ ('/bin/bash', '-c', $REMOTE_SET_HOST_KEY, '--', $hostnqn),
+ );
+ my $cmd = [
+ @ssh_cmd,
+ '-i',
+ ssh_key($scfg),
+ $target,
+ '--',
+ $remote_cmd,
+ ];
+
+ run_command(
+ $cmd,
+ input => "$key\n",
+ timeout => 10,
+ quiet => 1,
+ errmsg => "failed to configure NVMe DH-HMAC-CHAP key",
+ );
+}
+
+sub allow_host($scfg, $hostnqn) {
+ remote_call($scfg, 10, 'allow-host', $scfg->{subsysnqn}, $hostnqn);
+}
+
+sub reconcile($scfg) {
+ remote_call($scfg, 30, 'reconcile', $scfg->{subsysnqn}, $scfg->{pool});
+}
+
+sub delete_target($scfg) {
+ remote_call($scfg, 15, 'delete-target', $scfg->{subsysnqn});
+}
+
+my %lun_cmd_map = (
+ create_lu => sub($scfg, $timeout, $method, $device) {
+ return remote_call($scfg, $timeout, 'create', $scfg->{subsysnqn}, $scfg->{pool}, $device);
+ },
+ delete_lu => sub($scfg, $timeout, $method, $uuid) {
+ return remote_call($scfg, $timeout, 'delete', $scfg->{subsysnqn}, $uuid);
+ },
+ import_lu => sub($scfg, $timeout, $method, $device) {
+ return remote_call($scfg, $timeout, 'create', $scfg->{subsysnqn}, $scfg->{pool}, $device);
+ },
+ modify_lu => sub($scfg, $timeout, $method, $size, $uuid) {
+ return remote_call($scfg, $timeout, 'resize', $scfg->{subsysnqn}, $uuid);
+ },
+ add_view => sub($scfg, $timeout, $method, @params) {
+ return '';
+ },
+ list_view => sub($scfg, $timeout, $method, $uuid) {
+ return remote_call($scfg, $timeout, 'view', $scfg->{subsysnqn}, $scfg->{pool}, $uuid);
+ },
+ list_lu => sub($scfg, $timeout, $method, $device) {
+ return remote_call($scfg, $timeout, 'lookup', $scfg->{subsysnqn}, $scfg->{pool}, $device);
+ },
+);
+
+sub run_lun_command($scfg, $timeout, $method, @params) {
+ die "unknown command '$method'\n" if !exists($lun_cmd_map{$method});
+ return $lun_cmd_map{$method}->($scfg, $timeout, $method, @params);
+}
+
+1;
diff --git a/src/PVE/Storage/Makefile b/src/PVE/Storage/Makefile
index a67dc25..d1cbfe2 100644
--- a/src/PVE/Storage/Makefile
+++ b/src/PVE/Storage/Makefile
@@ -11,6 +11,7 @@ SOURCES= \
ISCSIDirectPlugin.pm \
ZFSPoolPlugin.pm \
ZFSPlugin.pm \
+ ZFSNVMePlugin.pm \
PBSPlugin.pm \
BTRFSPlugin.pm \
LvmThinPlugin.pm \
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 4f69f9b..3a45a73 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -35,7 +35,7 @@ our @COMMON_TAR_FLAGS = qw(
);
our @SHARED_STORAGE = (
- 'iscsi', 'nfs', 'cifs', 'rbd', 'cephfs', 'iscsidirect', 'zfs', 'drbd', 'pbs',
+ 'iscsi', 'nfs', 'cifs', 'rbd', 'cephfs', 'iscsidirect', 'zfs', 'zfsnvme', 'drbd', 'pbs',
);
our $QCOW2_PREALLOCATION = {
diff --git a/src/PVE/Storage/ZFSNVMePlugin.pm b/src/PVE/Storage/ZFSNVMePlugin.pm
new file mode 100644
index 0000000..83d086c
--- /dev/null
+++ b/src/PVE/Storage/ZFSNVMePlugin.pm
@@ -0,0 +1,716 @@
+package PVE::Storage::ZFSNVMePlugin;
+
+use v5.36;
+
+use File::Path qw(make_path);
+use IO::Socket::IP;
+use IO::File;
+use JSON;
+
+use PVE::JSONSchema;
+use PVE::RESTEnvironment qw(log_warn);
+use PVE::Storage::LunCmd::NVMET;
+use PVE::Storage::ZFSPlugin;
+use PVE::Tools qw(file_read_firstline file_set_contents run_command trim);
+
+use base qw(PVE::Storage::ZFSPlugin);
+
+my $nvme = '/usr/sbin/nvme';
+my $secret_dir = '/etc/pve/priv/storage';
+my $runtime_dir = '/run/pve-storage';
+my $max_paths = 16;
+
+my $RE_NQN = qr{
+ \A
+ nqn \.
+ [A-Za-z0-9] [A-Za-z0-9.-]*
+ :
+ [A-Za-z0-9] [A-Za-z0-9._:-]*
+ \z
+}nxx;
+my $RE_IPV4_PORTAL = qr{
+ \A
+ (?<address> [^:]+)
+ (?: : (?<port> [0-9]+))?
+ \z
+}nxx;
+my $RE_IPV6_PORTAL = qr{
+ \A
+ \[ (?<address> [^\]]+) \]
+ (?: : (?<port> [0-9]+))?
+ \z
+}nxx;
+my $RE_HOST_IFACE = qr{\A [A-Za-z0-9_.-]+ \z}nxx;
+my $RE_DHCHAP_KEY = qr{
+ \A DHHC-1 : [0-9A-Fa-f]{2} : [A-Za-z0-9+/=]+ : \z
+}nxx;
+my $RE_NVME_CONTROLLER = qr{\A nvme [0-9]+ \z}nxx;
+my $RE_NVME_SUBSYSTEM = qr{\A nvme-subsys [0-9]+ \z}nxx;
+my $RE_TRADDR = qr{(?: \A | ,) traddr=(?<value>[^,]+)}nxx;
+my $RE_TRSVCID = qr{(?: \A | ,) trsvcid=(?<value>[^,]+)}nxx;
+my $RE_HOST_IFACE_ADDRESS = qr{(?: \A | ,) host_iface=(?<value>[^,]+)}nxx;
+
+sub verify_nvme_nqn($value, $noerr = undef) {
+
+ if (
+ length($value) > 223
+ || $value !~ $RE_NQN
+ ) {
+ return undef if $noerr;
+ die "value is not a valid NVMe qualified name\n";
+ }
+
+ return $value;
+}
+
+sub parse_nvme_portals($value, $noerr = undef) {
+ my $result = [];
+ my $seen = {};
+
+ for my $entry (split(/,/, $value // '')) {
+ $entry = trim($entry);
+ my ($address, $port, $family);
+ if ($entry =~ $RE_IPV6_PORTAL) {
+ ($address, $port, $family) = ($+{address}, $+{port} // 4420, 'ipv6');
+ } elsif ($entry =~ $RE_IPV4_PORTAL) {
+ ($address, $port, $family) = ($+{address}, $+{port} // 4420, 'ipv4');
+ } else {
+ return undef if $noerr;
+ die "invalid NVMe/TCP portal '$entry'\n";
+ }
+
+ if (
+ !PVE::JSONSchema::pve_verify_ip($address, 1)
+ || ($family eq 'ipv4' && index($address, ':') >= 0)
+ || ($family eq 'ipv6' && index($address, ':') < 0)
+ || $port < 1
+ || $port > 65535
+ ) {
+ return undef if $noerr;
+ die "invalid NVMe/TCP portal '$entry'\n";
+ }
+ my $id = join("\0", $family, $address, $port);
+ if ($seen->{$id}++) {
+ return undef if $noerr;
+ die "duplicate NVMe/TCP portal '$entry'\n";
+ }
+ push $result->@*, {
+ address => $address,
+ port => int($port),
+ family => $family,
+ };
+ if (scalar($result->@*) > $max_paths) {
+ return undef if $noerr;
+ die "at most $max_paths NVMe/TCP portals are supported\n";
+ }
+ }
+
+ if (!$result->@*) {
+ return undef if $noerr;
+ die "at least one NVMe/TCP portal is required\n";
+ }
+
+ return $result;
+}
+
+my sub verify_nvme_portals($value, $noerr = undef) {
+ return undef if !parse_nvme_portals($value, $noerr);
+ return $value;
+}
+
+sub parse_nvme_host_ifaces($value, $noerr = undef) {
+ my $result = [];
+
+ for my $iface (split(/,/, $value // '')) {
+ $iface = trim($iface);
+ if (length($iface) < 1 || length($iface) > 15 || $iface !~ $RE_HOST_IFACE) {
+ return undef if $noerr;
+ die "invalid NVMe/TCP host interface '$iface'\n";
+ }
+ push $result->@*, $iface;
+ }
+
+ if (!$result->@*) {
+ return undef if $noerr;
+ die "at least one NVMe/TCP host interface is required\n";
+ }
+
+ return $result;
+}
+
+my sub verify_nvme_host_ifaces($value, $noerr = undef) {
+ return undef if !parse_nvme_host_ifaces($value, $noerr);
+ return $value;
+}
+
+sub _configured_portals($scfg) {
+ my $portals = parse_nvme_portals($scfg->{'nvme-portals'});
+ my $ifaces = parse_nvme_host_ifaces($scfg->{'nvme-host-ifaces'});
+ die "nvme-host-ifaces must contain one interface for each nvme-portals entry\n"
+ if scalar($ifaces->@*) != scalar($portals->@*);
+
+ for (my $i = 0; $i < scalar($portals->@*); $i++) {
+ $portals->[$i]->{host_iface} = $ifaces->[$i];
+ }
+ return $portals;
+}
+
+PVE::JSONSchema::register_format('pve-storage-nvme-nqn', \&verify_nvme_nqn);
+PVE::JSONSchema::register_format('pve-storage-nvme-portals', \&verify_nvme_portals);
+PVE::JSONSchema::register_format('pve-storage-nvme-host-ifaces', \&verify_nvme_host_ifaces);
+
+sub type($class) {
+ return 'zfsnvme';
+}
+
+sub plugindata($class) {
+ return {
+ content => [{ images => 1 }, { images => 1 }],
+ 'sensitive-properties' => { 'dhchap-key' => 1 },
+ };
+}
+
+sub properties($class) {
+ return {
+ subsysnqn => {
+ description => "NVMe subsystem qualified name.",
+ type => 'string',
+ format => 'pve-storage-nvme-nqn',
+ },
+ 'nvme-portals' => {
+ description =>
+ "Comma-separated NVMe/TCP target addresses. The default TCP port is 4420.",
+ type => 'string',
+ format => 'pve-storage-nvme-portals',
+ maxLength => 2048,
+ },
+ 'nvme-host-ifaces' => {
+ description =>
+ "Comma-separated local interfaces, in portal order. Interface names must be identical on every cluster node.",
+ type => 'string',
+ format => 'pve-storage-nvme-host-ifaces',
+ maxLength => 512,
+ },
+ 'dhchap-key' => {
+ description => "NVMe DH-HMAC-CHAP key in secret representation format.",
+ type => 'string',
+ maxLength => 256,
+ },
+ 'nvme-iopolicy' => {
+ description => "Native NVMe multipath I/O policy.",
+ type => 'string',
+ enum => ['numa', 'round-robin', 'queue-depth'],
+ default => 'round-robin',
+ },
+ 'nvme-keep-alive-tmo' => {
+ description => "NVMe keep-alive timeout in seconds.",
+ type => 'integer',
+ minimum => 1,
+ maximum => 120,
+ default => 5,
+ },
+ 'nvme-reconnect-delay' => {
+ description => "Delay between NVMe reconnect attempts in seconds.",
+ type => 'integer',
+ minimum => 1,
+ maximum => 120,
+ default => 2,
+ },
+ 'nvme-ctrl-loss-tmo' => {
+ description => "Time to keep retrying a lost NVMe controller in seconds.",
+ type => 'integer',
+ minimum => -1,
+ maximum => 86400,
+ default => 600,
+ },
+ 'nvme-nr-io-queues' => {
+ description => "Number of NVMe/TCP I/O queues per controller.",
+ type => 'integer',
+ minimum => 1,
+ maximum => 1024,
+ optional => 1,
+ },
+ };
+}
+
+sub options($class) {
+ return {
+ server => { fixed => 1 },
+ subsysnqn => { fixed => 1 },
+ 'nvme-portals' => { fixed => 1 },
+ 'nvme-host-ifaces' => { optional => 1 },
+ pool => { fixed => 1 },
+ blocksize => { fixed => 1 },
+ sparse => { optional => 1 },
+ 'dhchap-key' => { optional => 1 },
+ 'nvme-iopolicy' => { optional => 1 },
+ 'nvme-keep-alive-tmo' => { optional => 1 },
+ 'nvme-reconnect-delay' => { optional => 1 },
+ 'nvme-ctrl-loss-tmo' => { optional => 1 },
+ 'nvme-nr-io-queues' => { optional => 1 },
+ nodes => { optional => 1 },
+ disable => { optional => 1 },
+ content => { optional => 1 },
+ bwlimit => { optional => 1 },
+ };
+}
+
+sub check_config($class, $section_id, $config, $create, $skip_schema_check) {
+ if ($create) {
+ $config->{sparse} = 1 if !defined($config->{sparse});
+ $config->{'nvme-iopolicy'} //= 'round-robin';
+ $config->{'nvme-keep-alive-tmo'} //= 5;
+ $config->{'nvme-reconnect-delay'} //= 2;
+ $config->{'nvme-ctrl-loss-tmo'} //= 600;
+ }
+
+ verify_nvme_nqn($config->{subsysnqn}) if defined($config->{subsysnqn});
+ parse_nvme_portals($config->{'nvme-portals'}) if defined($config->{'nvme-portals'});
+ if (defined($config->{'nvme-host-ifaces'}) && defined($config->{'nvme-portals'})) {
+ _configured_portals($config);
+ } elsif (defined($config->{'nvme-host-ifaces'})) {
+ parse_nvme_host_ifaces($config->{'nvme-host-ifaces'});
+ }
+ return $class->SUPER::check_config($section_id, $config, $create, $skip_schema_check);
+}
+
+sub zfs_lun_provider($class, $scfg = undef) {
+ return 'PVE::Storage::LunCmd::NVMET';
+}
+
+sub zfs_request($class, $scfg, @params) {
+ local $scfg->{portal} = $scfg->{server};
+ return $class->SUPER::zfs_request($scfg, @params);
+}
+
+sub zfs_list_zvol($class, $scfg) {
+ my $list = $class->SUPER::zfs_list_zvol($scfg);
+ my $properties = $class->zfs_request(
+ $scfg,
+ 10,
+ 'get',
+ '-H',
+ '-d',
+ '1',
+ '-o',
+ 'name,value,source',
+ 'proxmox:nvme-subsys',
+ $scfg->{pool},
+ );
+ my $owned = {};
+ for my $line (split(/\n/, $properties)) {
+ my ($dataset, $nqn, $source) = split(/\t/, $line, 3);
+ next if !defined($source) || ($source ne 'local' && $source ne 'received');
+ next if !defined($nqn) || $nqn ne $scfg->{subsysnqn};
+ my $prefix = "$scfg->{pool}/";
+ next if index($dataset, $prefix) != 0;
+ $owned->{substr($dataset, length($prefix))} = 1;
+ }
+ for my $name (keys $list->%*) {
+ delete $list->{$name} if !$owned->{$name};
+ }
+ return $list;
+}
+
+my sub secret_path($storeid) {
+ return "$secret_dir/$storeid.nvme-dhchap";
+}
+
+sub _assert_unique_target($storeid, $scfg, $cfg = undef) {
+ $cfg //= PVE::Storage::config();
+ my $ids = $cfg->{ids} // {};
+ for my $other_id (keys $ids->%*) {
+ next if $other_id eq $storeid;
+ my $other = $cfg->{ids}->{$other_id};
+ next if ($other->{type} // '') ne 'zfsnvme';
+
+ die "NVMe subsystem NQN is already used by storage '$other_id'\n"
+ if ($other->{subsysnqn} // '') eq $scfg->{subsysnqn};
+ die "ZFS pool '$scfg->{pool}' on '$scfg->{server}' is already used by storage '$other_id'\n"
+ if ($other->{server} // '') eq $scfg->{server}
+ && ($other->{pool} // '') eq $scfg->{pool};
+ }
+}
+
+sub _validate_secret($key) {
+ die "missing NVMe DH-HMAC-CHAP key\n" if !defined($key) || $key eq '';
+ die "invalid NVMe DH-HMAC-CHAP key representation\n"
+ if $key !~ $RE_DHCHAP_KEY;
+ return $key;
+}
+
+my sub set_secret($storeid, $key) {
+ _validate_secret($key);
+ make_path($secret_dir, { mode => 0700 });
+ file_set_contents(secret_path($storeid), "$key\n", 0600);
+}
+
+my sub get_secret($storeid) {
+ my $key = file_read_firstline(secret_path($storeid));
+ return _validate_secret($key);
+}
+
+my sub delete_secret($storeid) {
+ unlink(secret_path($storeid));
+}
+
+sub on_add_hook($class, $storeid, $scfg, %sensitive) {
+ _configured_portals($scfg);
+ _assert_unique_target($storeid, $scfg);
+ set_secret($storeid, $sensitive{'dhchap-key'});
+ return;
+}
+
+sub on_update_hook_full($class, $storeid, $scfg, $update, $delete, $sensitive) {
+ my %prospective = ($scfg->%*, $update->%*);
+ delete @prospective{$delete->@*} if $delete;
+ verify_nvme_nqn($prospective{subsysnqn});
+ _configured_portals(\%prospective);
+ _assert_unique_target($storeid, \%prospective);
+
+ my $old_key = file_read_firstline(secret_path($storeid));
+ my $key = exists($sensitive->{'dhchap-key'}) ? $sensitive->{'dhchap-key'} : $old_key;
+ _validate_secret($key);
+
+ # nvmet stores authentication on the global Host NQN object, which can be
+ # shared by multiple subsystems. Replacing an active key in place can make
+ # unrelated storages unrecoverable on their next reconnect. Until PVE can
+ # coordinate a rolling rotation on every node, fail before mutating the
+ # cluster-wide secret.
+ die "online NVMe DH-HMAC-CHAP key rotation is not supported; create a new storage/NQN\n"
+ if defined($old_key) && $key ne $old_key;
+
+ set_secret($storeid, $key) if exists($sensitive->{'dhchap-key'});
+ return;
+}
+
+sub on_delete_hook($class, $storeid, $scfg) {
+ eval { $class->deactivate_storage($storeid, $scfg) };
+ log_warn("failed to disconnect NVMe storage '$storeid': $@") if $@;
+ eval { PVE::Storage::LunCmd::NVMET::delete_target($scfg) };
+ log_warn("failed to remove NVMe target for '$storeid': $@") if $@;
+ delete_secret($storeid);
+ return;
+}
+
+my sub runtime_config_path($storeid) {
+ return "$runtime_dir/nvme-$storeid.json";
+}
+
+my sub write_runtime_config($storeid, $scfg, $hostnqn, $hostid, $key, $portals) {
+ my $ports = [
+ map {
+ {
+ transport => 'tcp',
+ traddr => $_->{address},
+ host_iface => $_->{host_iface},
+ trsvcid => "$_->{port}",
+ keep_alive_tmo => $scfg->{'nvme-keep-alive-tmo'} // 5,
+ reconnect_delay => $scfg->{'nvme-reconnect-delay'} // 2,
+ ctrl_loss_tmo => $scfg->{'nvme-ctrl-loss-tmo'} // 600,
+ ($scfg->{'nvme-nr-io-queues'}
+ ? (nr_io_queues => $scfg->{'nvme-nr-io-queues'})
+ : ()),
+ }
+ } $portals->@*
+ ];
+ my $config = [
+ {
+ hostnqn => $hostnqn,
+ hostid => $hostid,
+ dhchap_key => $key,
+ subsystems => [
+ {
+ nqn => $scfg->{subsysnqn},
+ ports => $ports,
+ application => 'pve-storage',
+ },
+ ],
+ },
+ ];
+
+ make_path($runtime_dir, { mode => 0700 });
+ my $json = JSON->new->canonical->utf8->encode($config);
+ my $path = runtime_config_path($storeid);
+ file_set_contents($path, "$json\n", 0600);
+ return $path;
+}
+
+my sub controller_states($nqn) {
+ my $states = {};
+
+ opendir(my $dh, '/sys/class/nvme') or return $states;
+ while (defined(my $entry = readdir($dh))) {
+ next if $entry !~ $RE_NVME_CONTROLLER;
+ my $base = "/sys/class/nvme/$entry";
+ my $subsys = file_read_firstline("$base/subsysnqn");
+ next if !defined($subsys) || $subsys ne $nqn;
+ my $address = file_read_firstline("$base/address") // '';
+ my $state = file_read_firstline("$base/state") // 'unknown';
+ my $traddr = $address =~ $RE_TRADDR ? $+{value} : undef;
+ my $trsvcid = $address =~ $RE_TRSVCID ? $+{value} : undef;
+ my $host_iface = $address =~ $RE_HOST_IFACE_ADDRESS ? $+{value} : undef;
+ next if !defined($traddr) || !defined($trsvcid);
+ $states->{"$traddr:$trsvcid"} = {
+ device => "/dev/$entry",
+ host_iface => $host_iface,
+ state => $state,
+ };
+ }
+ closedir($dh);
+ return $states;
+}
+
+my sub portal_reachable($portal) {
+ my $socket = IO::Socket::IP->new(
+ PeerHost => $portal->{address},
+ PeerPort => $portal->{port},
+ Proto => 'tcp',
+ Timeout => 2,
+ );
+ return 0 if !$socket;
+ close($socket);
+ return 1;
+}
+
+sub _live_portal_count($states, $portals) {
+ my $live = 0;
+ for my $portal ($portals->@*) {
+ my $id = "$portal->{address}:$portal->{port}";
+ my $controller = $states->{$id};
+ $live++
+ if $controller
+ && $controller->{state} eq 'live'
+ && ($controller->{host_iface} // '') eq $portal->{host_iface};
+ }
+ return $live;
+}
+
+sub _connect_portal($config_path, $scfg, $portal) {
+ my $cmd = [
+ $nvme,
+ 'connect',
+ '--config',
+ $config_path,
+ '--transport',
+ 'tcp',
+ '--nqn',
+ $scfg->{subsysnqn},
+ '--traddr',
+ $portal->{address},
+ '--host-iface',
+ $portal->{host_iface},
+ '--trsvcid',
+ "$portal->{port}",
+ '--keep-alive-tmo',
+ $scfg->{'nvme-keep-alive-tmo'} // 5,
+ '--reconnect-delay',
+ $scfg->{'nvme-reconnect-delay'} // 2,
+ '--ctrl-loss-tmo',
+ $scfg->{'nvme-ctrl-loss-tmo'} // 600,
+ ];
+ if (my $queues = $scfg->{'nvme-nr-io-queues'}) {
+ push $cmd->@*, '--nr-io-queues', $queues;
+ }
+
+ run_command($cmd, timeout => 10, quiet => 1, errmsg => "NVMe/TCP connect failed");
+}
+
+my sub set_iopolicy($nqn, $policy) {
+ opendir(my $dh, '/sys/class/nvme-subsystem') or return;
+ while (defined(my $entry = readdir($dh))) {
+ next if $entry !~ $RE_NVME_SUBSYSTEM;
+ my $base = "/sys/class/nvme-subsystem/$entry";
+ my $subsys = file_read_firstline("$base/subsysnqn");
+ next if !defined($subsys) || $subsys ne $nqn;
+ my $fh = IO::File->new("$base/iopolicy", 'w')
+ or die "unable to set NVMe multipath policy: $!\n";
+ print $fh "$policy\n";
+ close($fh) or die "unable to set NVMe multipath policy: $!\n";
+ }
+ closedir($dh);
+}
+
+sub activate_storage($class, $storeid, $scfg, $cache = undef) {
+ $cache //= {};
+
+ die "nvme-cli is not installed\n" if !-x $nvme;
+ die "native NVMe multipath is disabled in the running kernel\n"
+ if (file_read_firstline('/sys/module/nvme_core/parameters/multipath') // 'N') ne 'Y';
+
+ _assert_unique_target($storeid, $scfg);
+ my $portals = _configured_portals($scfg);
+ my $states = controller_states($scfg->{subsysnqn});
+ my $force_reconcile =
+ delete($cache->{'zfsnvme-force-reconcile'}->{$storeid}) // 0;
+ my $live = _live_portal_count($states, $portals);
+
+ # This method is called by the periodic storage status loop. Once every
+ # configured controller is live, all lifecycle mutations are already
+ # reconciled by their individual operations, so avoid four SSH round-trips
+ # on every status poll. Missing namespaces explicitly force the slow path
+ # from activate_volume().
+ if (!$force_reconcile && $live == scalar($portals->@*)) {
+ die "missing NVMe DH-HMAC-CHAP key\n" if !-s secret_path($storeid);
+ set_iopolicy($scfg->{subsysnqn}, $scfg->{'nvme-iopolicy'} // 'round-robin');
+ return 1;
+ }
+
+ my $hostnqn = file_read_firstline('/etc/nvme/hostnqn')
+ // die "missing /etc/nvme/hostnqn\n";
+ my $hostid = file_read_firstline('/etc/nvme/hostid')
+ // die "missing /etc/nvme/hostid\n";
+ verify_nvme_nqn($hostnqn);
+ my $key = get_secret($storeid);
+ my $target_portals = [
+ map { "$_->{family},$_->{address},$_->{port}" } $portals->@*
+ ];
+
+ PVE::Storage::LunCmd::NVMET::ensure_target($scfg, $hostnqn, $target_portals);
+ PVE::Storage::LunCmd::NVMET::set_host_key($scfg, $hostnqn, $key);
+ PVE::Storage::LunCmd::NVMET::allow_host($scfg, $hostnqn);
+ PVE::Storage::LunCmd::NVMET::reconcile($scfg);
+
+ my $config_path =
+ write_runtime_config($storeid, $scfg, $hostnqn, $hostid, $key, $portals);
+ $states = controller_states($scfg->{subsysnqn});
+ for my $portal ($portals->@*) {
+ my $id = "$portal->{address}:$portal->{port}";
+ my $needs_rebind = 0;
+ if (my $controller = $states->{$id}) {
+ $needs_rebind = ($controller->{host_iface} // '') ne $portal->{host_iface};
+ next if !$needs_rebind && $controller->{state} ne 'dead';
+ run_command(
+ [$nvme, 'disconnect', '--device', $controller->{device}],
+ timeout => 5,
+ noerr => 1,
+ quiet => 1,
+ );
+ }
+ if (!portal_reachable($portal)) {
+ log_warn("NVMe/TCP portal '$id' is unreachable");
+ next;
+ }
+ eval { _connect_portal($config_path, $scfg, $portal) };
+ log_warn("$id: $@") if $@;
+
+ # Rebind controllers one at a time. Do not disconnect the next path
+ # until this one is live on its configured interface, so changing an
+ # interface mapping cannot cause an avoidable all-path outage.
+ if ($needs_rebind) {
+ my $rebound = 0;
+ for (my $attempt = 0; $attempt < 40; $attempt++) {
+ $states = controller_states($scfg->{subsysnqn});
+ my $controller = $states->{$id};
+ if (
+ $controller
+ && $controller->{state} eq 'live'
+ && ($controller->{host_iface} // '') eq $portal->{host_iface}
+ ) {
+ $rebound = 1;
+ last;
+ }
+ select(undef, undef, undef, 0.25);
+ }
+ if (!$rebound) {
+ log_warn("NVMe/TCP portal '$id' did not rebind to '$portal->{host_iface}'");
+ last;
+ }
+ }
+ }
+
+ # Existing controllers can be in the middle of their kernel reconnect delay
+ # after a target restart. Do not create duplicates, but give that recovery
+ # cycle enough time to complete before declaring the storage unavailable.
+ for (my $attempt = 0; $attempt < 60; $attempt++) {
+ $states = controller_states($scfg->{subsysnqn});
+ last if _live_portal_count($states, $portals);
+ select(undef, undef, undef, 0.25);
+ }
+ $live = _live_portal_count($states, $portals);
+ die "no live NVMe/TCP path for storage '$storeid'\n" if !$live;
+ log_warn("storage '$storeid' is degraded: $live/" . scalar($portals->@*) . " paths live")
+ if $live < scalar($portals->@*);
+
+ set_iopolicy($scfg->{subsysnqn}, $scfg->{'nvme-iopolicy'} // 'round-robin');
+ return 1;
+}
+
+sub deactivate_storage($class, $storeid, $scfg, $cache = undef) {
+
+ run_command(
+ [$nvme, 'disconnect', '--nqn', $scfg->{subsysnqn}],
+ timeout => 15,
+ noerr => 1,
+ quiet => 1,
+ ) if -x $nvme;
+ unlink(runtime_config_path($storeid));
+ return 1;
+}
+
+sub path($class, $scfg, $volname, $storeid, $snapname = undef) {
+ die "direct access to snapshots not implemented\n" if defined($snapname);
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+ my $uuid = $class->zfs_get_lu_name($scfg, $name);
+ my $path = "/dev/disk/by-id/nvme-uuid.$uuid";
+ return ($path, $vmid, $vtype);
+}
+
+sub qemu_blockdev_options($class, $scfg, $storeid, $volname, $machine_version, $options) {
+ die "direct access to snapshots not implemented\n" if $options->{'snapshot-name'};
+ my ($path) = $class->path($scfg, $volname, $storeid);
+ return { driver => 'host_device', filename => $path };
+}
+
+sub activate_volume($class, $storeid, $scfg, $volname, $snapname, $cache = undef) {
+ die "unable to activate snapshot from remote zfs storage\n" if $snapname;
+ my ($path) = $class->path($scfg, $volname, $storeid);
+ if (!-b $path) {
+ $cache //= {};
+ $cache->{'zfsnvme-force-reconcile'}->{$storeid} = 1;
+ $class->activate_storage($storeid, $scfg, $cache);
+ for (my $attempt = 0; $attempt < 40 && !-b $path; $attempt++) {
+ select(undef, undef, undef, 0.25);
+ }
+ }
+ die "NVMe namespace for '$volname' did not appear\n" if !-b $path;
+ return 1;
+}
+
+sub deactivate_volume($class, $storeid, $scfg, $volname, $snapname, $cache = undef) {
+ die "unable to deactivate snapshot from remote zfs storage\n" if $snapname;
+ return 1;
+}
+
+sub volume_resize($class, $scfg, $storeid, $volname, $size, $running, $snapname) {
+ # QEMU's block_resize command explicitly cannot resize host block devices.
+ # Reject before changing the zvol/namespace, otherwise qemu-server would
+ # leave the backend larger while the running VM and its config keep the old
+ # size. A stopped VM is reopened with the new capacity on its next start.
+ die "online resize is not supported for NVMe/TCP block devices; stop the VM first\n"
+ if $running;
+ return $class->SUPER::volume_resize(
+ $scfg, $storeid, $volname, $size, $running, $snapname,
+ );
+}
+
+sub alloc_image($class, $storeid, $scfg, $vmid, $fmt, $name, $size) {
+ die "unsupported format '$fmt'" if $fmt ne 'raw';
+ die "illegal name '$name' - should be 'vm-$vmid-*'\n"
+ if $name && index($name, "vm-$vmid-") != 0;
+ my $volname = $name // $class->find_free_diskname($storeid, $scfg, $vmid, $fmt);
+
+ $class->zfs_create_zvol($scfg, $volname, $size);
+ eval {
+ my $uuid = $class->zfs_create_lu($scfg, $volname);
+ $class->zfs_add_lun_mapping_entry($scfg, $volname, $uuid);
+ };
+ if (my $err = $@) {
+ eval { $class->zfs_delete_zvol($scfg, $volname) };
+ warn "failed to clean up zvol '$volname': $@" if $@;
+ die $err;
+ }
+ return $volname;
+}
+
+1;
diff --git a/src/test/run_plugin_tests.pl b/src/test/run_plugin_tests.pl
index 8bce9d3..c341f4d 100755
--- a/src/test/run_plugin_tests.pl
+++ b/src/test/run_plugin_tests.pl
@@ -17,6 +17,7 @@ my $res = $harness->runtests(
"get_subdir_test.pm",
"filesystem_path_test.pm",
"prune_backups_test.pm",
+ "zfsnvme_test.pm",
);
exit -1 if !$res || $res->{failed} || $res->{parse_errors};
diff --git a/src/test/zfsnvme_test.pm b/src/test/zfsnvme_test.pm
new file mode 100644
index 0000000..3e9e04b
--- /dev/null
+++ b/src/test/zfsnvme_test.pm
@@ -0,0 +1,350 @@
+package PVE::Storage::TestZFSNVMe;
+
+use v5.36;
+
+use lib qw(..);
+
+use PVE::Storage::LunCmd::LIO;
+use PVE::Storage::LunCmd::NVMET;
+use PVE::Storage::ZFSNVMePlugin;
+use PVE::Storage::ZFSPlugin;
+use Test::MockModule;
+use Test::More;
+
+is(
+ PVE::Storage::ZFSNVMePlugin::verify_nvme_nqn(
+ 'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+ ),
+ 'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+ 'accepts a standards-based NQN',
+);
+
+ok(
+ !PVE::Storage::ZFSNVMePlugin::verify_nvme_nqn('not-an-nqn', 1),
+ 'rejects an invalid NQN',
+);
+
+is_deeply(
+ PVE::Storage::ZFSNVMePlugin::parse_nvme_portals(
+ '10.90.1.11:4420,[fd00::11]:4421,10.90.2.11',
+ ),
+ [
+ { address => '10.90.1.11', port => 4420, family => 'ipv4' },
+ { address => 'fd00::11', port => 4421, family => 'ipv6' },
+ { address => '10.90.2.11', port => 4420, family => 'ipv4' },
+ ],
+ 'parses and normalizes IPv4 and IPv6 portals',
+);
+
+eval {
+ PVE::Storage::ZFSNVMePlugin::parse_nvme_portals(
+ '10.90.1.11:4420,10.90.1.11:4420',
+ );
+};
+like($@, qr/duplicate NVMe\/TCP portal/, 'duplicate portal is rejected');
+
+ok(
+ !PVE::Storage::ZFSNVMePlugin::parse_nvme_portals('10.90.1.11:70000', 1),
+ 'rejects an invalid TCP port',
+);
+
+ok(
+ !PVE::Storage::ZFSNVMePlugin::parse_nvme_portals(
+ join(',', map { "10.90.1.$_:4420" } 1 .. 17),
+ 1,
+ ),
+ 'limits the number of configured paths',
+);
+
+is_deeply(
+ PVE::Storage::ZFSNVMePlugin::_configured_portals({
+ 'nvme-portals' => '10.90.1.11:4420,10.90.2.11:4421',
+ 'nvme-host-ifaces' => 'ens20,ens21',
+ }),
+ [
+ {
+ address => '10.90.1.11',
+ port => 4420,
+ family => 'ipv4',
+ host_iface => 'ens20',
+ },
+ {
+ address => '10.90.2.11',
+ port => 4421,
+ family => 'ipv4',
+ host_iface => 'ens21',
+ },
+ ],
+ 'binds each target portal to its configured local interface',
+);
+
+eval {
+ PVE::Storage::ZFSNVMePlugin::_configured_portals({
+ 'nvme-portals' => '10.90.1.11,10.90.2.11',
+ 'nvme-host-ifaces' => 'ens20',
+ });
+};
+like($@, qr/one interface for each/, 'portal and host-interface counts must match');
+
+eval {
+ PVE::Storage::ZFSNVMePlugin::_assert_unique_target(
+ 'new-storage',
+ {
+ server => '192.0.2.10',
+ pool => 'tank/new',
+ subsysnqn => 'nqn.2026-07.example:duplicate',
+ },
+ {
+ ids => {
+ existing => {
+ type => 'zfsnvme',
+ server => '192.0.2.11',
+ pool => 'tank/existing',
+ subsysnqn => 'nqn.2026-07.example:duplicate',
+ },
+ },
+ },
+ );
+};
+like($@, qr/NQN is already used/, 'a subsystem NQN cannot be shared by storage definitions');
+
+eval {
+ PVE::Storage::ZFSNVMePlugin::_assert_unique_target(
+ 'new-storage',
+ {
+ server => '192.0.2.10',
+ pool => 'tank/shared',
+ subsysnqn => 'nqn.2026-07.example:new',
+ },
+ {
+ ids => {
+ existing => {
+ type => 'zfsnvme',
+ server => '192.0.2.10',
+ pool => 'tank/shared',
+ subsysnqn => 'nqn.2026-07.example:existing',
+ },
+ },
+ },
+ );
+};
+like($@, qr/ZFS pool .* is already used/, 'a target pool cannot be shared by storage definitions');
+
+ok(
+ !PVE::Storage::ZFSNVMePlugin::parse_nvme_host_ifaces('ens20,not/an/interface', 1),
+ 'rejects an invalid host interface name',
+);
+
+is(
+ PVE::Storage::ZFSNVMePlugin::_live_portal_count(
+ {
+ '10.90.1.11:4420' => { state => 'live', host_iface => 'eth0' },
+ '10.90.2.11:4420' => { state => 'live', host_iface => 'ens21' },
+ },
+ [
+ { address => '10.90.1.11', port => 4420, host_iface => 'ens20' },
+ { address => '10.90.2.11', port => 4420, host_iface => 'ens21' },
+ ],
+ ),
+ 1,
+ 'a live controller on the wrong interface is not an eligible path',
+);
+
+is(
+ PVE::Storage::ZFSNVMePlugin::_validate_secret(
+ 'DHHC-1:01:YWJjZGVmZ2hpamtsbW5vcA==:',
+ ),
+ 'DHHC-1:01:YWJjZGVmZ2hpamtsbW5vcA==:',
+ 'accepts an NVMe DH-HMAC-CHAP secret representation',
+);
+
+eval { PVE::Storage::ZFSNVMePlugin::_validate_secret('plaintext') };
+like($@, qr/invalid NVMe DH-HMAC-CHAP/, 'rejects a plaintext secret');
+
+my $nvme_mock = Test::MockModule->new('PVE::Storage::ZFSNVMePlugin');
+$nvme_mock->redefine(zfs_get_lu_name => sub { return '12345678-1234-1234-1234-123456789abc' });
+
+my $scfg = {};
+my ($path, $vmid, $vtype) = PVE::Storage::ZFSNVMePlugin->path(
+ $scfg,
+ 'vm-100-disk-0',
+ 'nvmetest',
+);
+is(
+ $path,
+ '/dev/disk/by-id/nvme-uuid.12345678-1234-1234-1234-123456789abc',
+ 'uses the stable namespace UUID symlink',
+);
+is($vmid, 100, 'returns the VM owner');
+is($vtype, 'images', 'returns the volume type');
+is_deeply(
+ PVE::Storage::ZFSNVMePlugin->qemu_blockdev_options(
+ $scfg,
+ 'nvmetest',
+ 'vm-100-disk-0',
+ undef,
+ {},
+ ),
+ {
+ driver => 'host_device',
+ filename => '/dev/disk/by-id/nvme-uuid.12345678-1234-1234-1234-123456789abc',
+ },
+ 'uses the QEMU host_device driver',
+);
+
+my $lio_mock = Test::MockModule->new('PVE::Storage::LunCmd::LIO');
+my @provider_args;
+$lio_mock->redefine(
+ run_lun_command => sub {
+ @provider_args = @_;
+ return 'ok';
+ },
+);
+is(
+ PVE::Storage::ZFSPlugin->zfs_request(
+ { iscsiprovider => 'LIO' },
+ 10,
+ 'add_view',
+ 'guid',
+ ),
+ 'ok',
+ 'legacy LIO dispatch still works',
+);
+is(ref($provider_args[0]), 'HASH', 'provider dispatch does not inject a class argument');
+is($provider_args[1], 10, 'provider timeout argument is preserved');
+is($provider_args[2], 'add_view', 'provider method argument is preserved');
+is($provider_args[3], 'guid', 'provider parameters are preserved');
+
+my @connect_cmd;
+$nvme_mock->redefine(
+ run_command => sub {
+ @connect_cmd = $_[0]->@*;
+ return 0;
+ },
+);
+PVE::Storage::ZFSNVMePlugin::_connect_portal(
+ '/run/pve-storage/test.json',
+ { subsysnqn => 'nqn.2026-07.example:test' },
+ { address => '10.90.2.11', port => 4420, host_iface => 'ens21' },
+);
+is_deeply(
+ [@connect_cmd[10, 11]],
+ ['--host-iface', 'ens21'],
+ 'nvme connect is explicitly bound to the configured data interface',
+);
+
+$nvme_mock->redefine(
+ zfs_request => sub {
+ my ($class, $config, $timeout, $method, @params) = @_;
+ return join(
+ "\n",
+ "tank/vm-100-disk-0\t1048576\t-\tvolume\t-",
+ "tank/vm-200-disk-0\t1048576\t-\tvolume\t-",
+ "tank/vm-300-disk-0\t1048576\t-\tvolume\t-",
+ '',
+ ) if $method eq 'list';
+ return join(
+ "\n",
+ "tank/vm-100-disk-0\tnqn.2026-07.example:owned\tlocal",
+ "tank/vm-200-disk-0\t-\t-",
+ "tank/vm-300-disk-0\tnqn.2026-07.example:owned\tinherited from tank",
+ '',
+ ) if $method eq 'get';
+ die "unexpected mocked ZFS method '$method'\n";
+ },
+);
+my $owned_zvols = PVE::Storage::ZFSNVMePlugin->zfs_list_zvol(
+ { pool => 'tank', subsysnqn => 'nqn.2026-07.example:owned' },
+);
+is_deeply(
+ [sort keys $owned_zvols->%*],
+ ['vm-100-disk-0'],
+ 'volume listing requires a local or received ownership property',
+);
+
+eval {
+ PVE::Storage::ZFSNVMePlugin->volume_resize(
+ {}, 'nvmetest', 'vm-100-disk-0', 2 * 1024 * 1024 * 1024, 1, undef,
+ );
+};
+like(
+ $@,
+ qr/online resize is not supported.*stop the VM first/,
+ 'online resize is rejected before mutating the backend',
+);
+
+my $update_key = 'DHHC-1:01:dXBkYXRlLXRlc3Qta2V5:';
+$nvme_mock->redefine(file_read_firstline => sub { return $update_key });
+my $zfs_parent_mock = Test::MockModule->new('PVE::Storage::ZFSPlugin');
+$zfs_parent_mock->redefine(check_config => sub { return $_[2] });
+eval {
+ PVE::Storage::ZFSNVMePlugin->check_config(
+ 'nvmetest',
+ { 'nvme-host-ifaces' => 'ens20,ens21' },
+ 0,
+ 1,
+ );
+};
+is($@, '', 'check_config accepts a valid partial update');
+
+{
+ no warnings 'redefine';
+ local *PVE::Storage::config = sub { return { ids => {} } };
+ eval {
+ PVE::Storage::ZFSNVMePlugin->on_update_hook_full(
+ 'nvmetest',
+ {
+ subsysnqn => 'nqn.2026-07.example:test',
+ 'nvme-portals' => '10.90.1.11,10.90.2.11',
+ },
+ { 'nvme-host-ifaces' => 'ens20,ens21' },
+ undef,
+ {},
+ );
+ };
+ is($@, '', 'partial update with no delete list validates against the current config');
+}
+
+my $nvmet_mock = Test::MockModule->new('PVE::Storage::LunCmd::NVMET');
+my @helper_calls;
+$nvmet_mock->redefine(
+ run_command => sub {
+ my ($cmd, %opts) = @_;
+ push @helper_calls, { cmd => $cmd, %opts };
+ return 0;
+ },
+);
+PVE::Storage::LunCmd::NVMET::ensure_target(
+ {
+ server => '192.0.2.10',
+ subsysnqn => 'nqn.2026-07.example:test',
+ pool => 'tank/pve-nvme',
+ },
+ 'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+ ['ipv4,10.90.1.11,4420'],
+);
+like(
+ $helper_calls[0]->{input},
+ qr/current_model.*current_serial.*refusing to take over existing NVMe subsystem/s,
+ 'target reconcile verifies model and deterministic serial before taking over a subsystem',
+);
+
+my ($key_cmd, %key_opts);
+$nvmet_mock->redefine(
+ run_command => sub {
+ ($key_cmd, %key_opts) = @_;
+ return 0;
+ },
+);
+my $test_key = 'DHHC-1:01:bm90LWEtcmVhbC1rZXk=:';
+PVE::Storage::LunCmd::NVMET::set_host_key(
+ { server => '192.0.2.10' },
+ 'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+ $test_key,
+);
+unlike(join(' ', $key_cmd->@*), qr/\Q$test_key\E/, 'DHCHAP key is absent from the process argv');
+is($key_opts{input}, "$test_key\n", 'DHCHAP key is provided through standard input');
+
+done_testing();
+
+1;
--
2.54.0.windows.1
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH storage v2 7/7] zfsnvme: restore ACLs before publishing target
2026-08-02 3:31 [PATCH storage v2 0/7] add native ZFS over NVMe/TCP backend Joaquin Varela
` (5 preceding siblings ...)
2026-08-02 3:31 ` [PATCH storage v2 6/7] zfsnvme: accept short volume activation calls Joaquin Varela
@ 2026-08-02 3:31 ` Joaquin Varela
6 siblings, 0 replies; 8+ messages in thread
From: Joaquin Varela @ 2026-08-02 3:31 UTC (permalink / raw)
To: pve-devel; +Cc: Joaquin Varela
Keep the subsystem unreachable while reconstructing derived configfs
state. Restore every configured Host NQN, its DH-HMAC-CHAP key and
namespace before linking the subsystem into any NVMe/TCP port.
This prevents reconnecting initiators from treating a transient host-not-
allowed response as a terminal controller failure after a target reboot.
Signed-off-by: Joaquin Varela <joaquinvarela@neatech.ar>
---
src/PVE/Storage/LunCmd/NVMET.pm | 47 +++++++++++++++++++++-----
src/PVE/Storage/ZFSNVMePlugin.pm | 58 ++++++++++++++++++++++++++++++--
src/test/zfsnvme_test.pm | 40 +++++++++++++++++++++-
3 files changed, 132 insertions(+), 13 deletions(-)
diff --git a/src/PVE/Storage/LunCmd/NVMET.pm b/src/PVE/Storage/LunCmd/NVMET.pm
index 7a902da..c5b9203 100644
--- a/src/PVE/Storage/LunCmd/NVMET.pm
+++ b/src/PVE/Storage/LunCmd/NVMET.pm
@@ -112,8 +112,7 @@ allocate_port_id() {
}
ensure_port() {
- local nqn="$1"
- local spec="$2"
+ local spec="$1"
local family address service id port
IFS=, read -r family address service <<<"$spec"
@@ -137,10 +136,6 @@ ensure_port() {
printf '%s\n' "$service" >"$port/addr_trsvcid"
fi
- port="$ROOT/ports/$id"
- if [[ ! -e "$port/subsystems/$nqn" ]]; then
- ln -s "$ROOT/subsystems/$nqn" "$port/subsystems/$nqn"
- fi
printf '%s\n' "$id"
}
@@ -148,7 +143,7 @@ ensure_target() {
local nqn="$1"
local pool="$2"
shift 2
- local spec id port desired_ports=''
+ local spec
validate_nqn "$nqn"
validate_pool "$pool"
@@ -156,7 +151,31 @@ ensure_target() {
ensure_subsystem "$nqn"
for spec in "$@"; do
- id="$(ensure_port "$nqn" "$spec")"
+ ensure_port "$spec" >/dev/null
+ done
+}
+
+publish_target() {
+ local nqn="$1"
+ shift
+ local spec id port desired_ports=''
+
+ validate_nqn "$nqn"
+ (($# >= 1)) || die "at least one NVMe/TCP portal is required"
+ [[ -d "$ROOT/subsystems/$nqn" ]] || die "NVMe subsystem does not exist"
+
+ # The subsystem only becomes reachable after every namespace, host ACL and
+ # authentication key has been restored. Publishing it earlier makes an
+ # initiator treat a transient "host not allowed" response as permanent and
+ # remove the controller, failing queued I/O despite ctrl_loss_tmo.
+ for spec in "$@"; do
+ IFS=, read -r family address service <<<"$spec"
+ id="$(find_port "$family" "$address" "$service" || true)"
+ [[ -n "$id" ]] || die "NVMe/TCP portal '$spec' is not configured"
+ port="$ROOT/ports/$id"
+ if [[ ! -e "$port/subsystems/$nqn" ]]; then
+ ln -s "$ROOT/subsystems/$nqn" "$port/subsystems/$nqn"
+ fi
desired_ports="$desired_ports $id"
done
@@ -523,6 +542,7 @@ shift || true
case "$mode" in
ensure-target) ensure_target "$@" ;;
+ publish-target) publish_target "$@" ;;
ensure-host) ensure_host "$@" ;;
allow-host) allow_host "$@" ;;
create) create_volume "$@" ;;
@@ -616,10 +636,15 @@ sub get_base($scfg) {
return '/dev/zvol';
}
-sub ensure_target($scfg, $hostnqn, $portals) {
+sub ensure_target($scfg, $portals) {
my $nqn = $scfg->{subsysnqn};
remote_call($scfg, 15, 'ensure-target', $nqn, $scfg->{pool}, $portals->@*);
+}
+
+sub ensure_host($scfg, $hostnqn) {
+ my $nqn = $scfg->{subsysnqn};
+
remote_call($scfg, 10, 'ensure-host', $nqn, $hostnqn);
}
@@ -657,6 +682,10 @@ sub reconcile($scfg) {
remote_call($scfg, 30, 'reconcile', $scfg->{subsysnqn}, $scfg->{pool});
}
+sub publish_target($scfg, $portals) {
+ remote_call($scfg, 15, 'publish-target', $scfg->{subsysnqn}, $portals->@*);
+}
+
sub delete_target($scfg) {
remote_call($scfg, 15, 'delete-target', $scfg->{subsysnqn});
}
diff --git a/src/PVE/Storage/ZFSNVMePlugin.pm b/src/PVE/Storage/ZFSNVMePlugin.pm
index 54c406d..eedc67d 100644
--- a/src/PVE/Storage/ZFSNVMePlugin.pm
+++ b/src/PVE/Storage/ZFSNVMePlugin.pm
@@ -19,6 +19,7 @@ my $nvme = '/usr/sbin/nvme';
my $secret_dir = '/etc/pve/priv/storage';
my $runtime_dir = '/run/pve-storage';
my $max_paths = 16;
+my $max_hosts = 64;
my $RE_NQN = qr{
\A
@@ -140,11 +141,41 @@ sub parse_nvme_host_ifaces($value, $noerr = undef) {
return $result;
}
+sub parse_nvme_host_nqns($value, $noerr = undef) {
+ my $result = [];
+ my $seen = {};
+
+ for my $hostnqn (split(/,/, $value // '')) {
+ $hostnqn = trim($hostnqn);
+ if (!verify_nvme_nqn($hostnqn, 1) || $seen->{$hostnqn}++) {
+ return undef if $noerr;
+ die "invalid or duplicate NVMe host NQN '$hostnqn'\n";
+ }
+ push $result->@*, $hostnqn;
+ if (scalar($result->@*) > $max_hosts) {
+ return undef if $noerr;
+ die "at most $max_hosts NVMe host NQNs are supported\n";
+ }
+ }
+
+ if (!$result->@*) {
+ return undef if $noerr;
+ die "at least one NVMe host NQN is required\n";
+ }
+
+ return $result;
+}
+
my sub verify_nvme_host_ifaces($value, $noerr = undef) {
return undef if !parse_nvme_host_ifaces($value, $noerr);
return $value;
}
+my sub verify_nvme_host_nqns($value, $noerr = undef) {
+ return undef if !parse_nvme_host_nqns($value, $noerr);
+ return $value;
+}
+
sub _configured_portals($scfg) {
my $portals = parse_nvme_portals($scfg->{'nvme-portals'});
my $ifaces = parse_nvme_host_ifaces($scfg->{'nvme-host-ifaces'});
@@ -172,6 +203,7 @@ sub _validate_local_ifaces($portals) {
PVE::JSONSchema::register_format('pve-storage-nvme-nqn', \&verify_nvme_nqn);
PVE::JSONSchema::register_format('pve-storage-nvme-portals', \&verify_nvme_portals);
PVE::JSONSchema::register_format('pve-storage-nvme-host-ifaces', \&verify_nvme_host_ifaces);
+PVE::JSONSchema::register_format('pve-storage-nvme-host-nqns', \&verify_nvme_host_nqns);
sub type($class) {
return 'zfsnvme';
@@ -205,6 +237,13 @@ sub properties($class) {
format => 'pve-storage-nvme-host-ifaces',
maxLength => 512,
},
+ 'nvme-host-nqns' => {
+ description =>
+ "Comma-separated /etc/nvme/hostnqn values for every cluster node allowed to use this storage.",
+ type => 'string',
+ format => 'pve-storage-nvme-host-nqns',
+ maxLength => 8192,
+ },
'dhchap-key' => {
description => "NVMe DH-HMAC-CHAP key in secret representation format.",
type => 'string',
@@ -261,6 +300,7 @@ sub options($class) {
subsysnqn => { fixed => 1 },
'nvme-portals' => { fixed => 1 },
'nvme-host-ifaces' => { optional => 1 },
+ 'nvme-host-nqns' => { optional => 1 },
pool => { fixed => 1 },
blocksize => { fixed => 1 },
sparse => { optional => 1 },
@@ -306,6 +346,8 @@ sub check_config($class, $section_id, $config, $create, $skip_schema_check) {
} elsif (defined($config->{'nvme-host-ifaces'})) {
parse_nvme_host_ifaces($config->{'nvme-host-ifaces'});
}
+ parse_nvme_host_nqns($config->{'nvme-host-nqns'})
+ if defined($config->{'nvme-host-nqns'});
_validate_fail_fast_timeout($config, $create ? 600 : undef);
return $class->SUPER::check_config($section_id, $config, $create, $skip_schema_check);
}
@@ -392,6 +434,7 @@ my sub delete_secret($storeid) {
sub on_add_hook($class, $storeid, $scfg, %sensitive) {
_configured_portals($scfg);
+ parse_nvme_host_nqns($scfg->{'nvme-host-nqns'});
_assert_unique_target($storeid, $scfg);
set_secret($storeid, $sensitive{'dhchap-key'});
return;
@@ -402,6 +445,7 @@ sub on_update_hook_full($class, $storeid, $scfg, $update, $delete, $sensitive) {
delete @prospective{$delete->@*} if $delete;
verify_nvme_nqn($prospective{subsysnqn});
_configured_portals(\%prospective);
+ parse_nvme_host_nqns($prospective{'nvme-host-nqns'});
_validate_fail_fast_timeout(\%prospective, 600);
_assert_unique_target($storeid, \%prospective);
@@ -667,15 +711,23 @@ sub activate_storage($class, $storeid, $scfg, $cache = undef) {
my $hostid = file_read_firstline('/etc/nvme/hostid')
// die "missing /etc/nvme/hostid\n";
verify_nvme_nqn($hostnqn);
+ my $hostnqns = parse_nvme_host_nqns($scfg->{'nvme-host-nqns'});
+ my $local_host_is_allowed = grep { $_ eq $hostnqn } $hostnqns->@*;
+ die "local NVMe host NQN '$hostnqn' is missing from nvme-host-nqns\n"
+ if !$local_host_is_allowed;
my $key = get_secret($storeid);
my $target_portals = [
map { "$_->{family},$_->{address},$_->{port}" } $portals->@*
];
- PVE::Storage::LunCmd::NVMET::ensure_target($scfg, $hostnqn, $target_portals);
- PVE::Storage::LunCmd::NVMET::set_host_key($scfg, $hostnqn, $key);
- PVE::Storage::LunCmd::NVMET::allow_host($scfg, $hostnqn);
+ PVE::Storage::LunCmd::NVMET::ensure_target($scfg, $target_portals);
+ for my $allowed_hostnqn ($hostnqns->@*) {
+ PVE::Storage::LunCmd::NVMET::ensure_host($scfg, $allowed_hostnqn);
+ PVE::Storage::LunCmd::NVMET::set_host_key($scfg, $allowed_hostnqn, $key);
+ PVE::Storage::LunCmd::NVMET::allow_host($scfg, $allowed_hostnqn);
+ }
PVE::Storage::LunCmd::NVMET::reconcile($scfg);
+ PVE::Storage::LunCmd::NVMET::publish_target($scfg, $target_portals);
my $config_path =
write_runtime_config($storeid, $scfg, $hostnqn, $hostid, $key, $portals);
diff --git a/src/test/zfsnvme_test.pm b/src/test/zfsnvme_test.pm
index d64b149..3d69ec7 100644
--- a/src/test/zfsnvme_test.pm
+++ b/src/test/zfsnvme_test.pm
@@ -26,6 +26,18 @@ ok(
'rejects an invalid NQN',
);
+my $hostnqn_a = 'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc';
+my $hostnqn_b = 'nqn.2014-08.org.nvmexpress:uuid:abcdefab-abcd-abcd-abcd-abcdefabcdef';
+is_deeply(
+ PVE::Storage::ZFSNVMePlugin::parse_nvme_host_nqns("$hostnqn_a,$hostnqn_b"),
+ [$hostnqn_a, $hostnqn_b],
+ 'parses the complete cluster NVMe host allow-list',
+);
+ok(
+ !PVE::Storage::ZFSNVMePlugin::parse_nvme_host_nqns("$hostnqn_a,$hostnqn_a", 1),
+ 'rejects duplicate NVMe host NQNs',
+);
+
is_deeply(
PVE::Storage::ZFSNVMePlugin::parse_nvme_portals(
'10.90.1.11:4420,[fd00::11]:4421,10.90.2.11',
@@ -421,6 +433,7 @@ is($@, '', 'fast I/O fail remains valid with infinite controller reconnect');
{
subsysnqn => 'nqn.2026-07.example:test',
'nvme-portals' => '10.90.1.11,10.90.2.11',
+ 'nvme-host-nqns' => $hostnqn_a,
},
{ 'nvme-host-ifaces' => 'ens20,ens21' },
undef,
@@ -445,7 +458,6 @@ PVE::Storage::LunCmd::NVMET::ensure_target(
subsysnqn => 'nqn.2026-07.example:test',
pool => 'tank/pve-nvme',
},
- 'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
['ipv4,10.90.1.11,4420'],
);
like(
@@ -453,6 +465,32 @@ like(
qr/current_model.*current_serial.*refusing to take over existing NVMe subsystem/s,
'target reconcile verifies model and deterministic serial before taking over a subsystem',
);
+my ($ensure_port_body) = $helper_calls[0]->{input} =~ /^ensure_port\(\) \{\n(?<body>.*?)^\}/ms;
+ok(defined($ensure_port_body), 'remote helper contains the port preparation function');
+unlike(
+ $ensure_port_body,
+ qr{ln[ ]-s},
+ 'target setup does not publish a subsystem before ACL and namespace reconciliation',
+);
+like(
+ $helper_calls[0]->{input},
+ qr{^publish_target\(\).*?ln[ ]-s}ms,
+ 'the remote helper exposes the subsystem only in its publish operation',
+);
+
+@helper_calls = ();
+PVE::Storage::LunCmd::NVMET::publish_target(
+ {
+ server => '192.0.2.10',
+ subsysnqn => 'nqn.2026-07.example:test',
+ },
+ ['ipv4,10.90.1.11,4420'],
+);
+like(
+ join(' ', $helper_calls[0]->{cmd}->@*),
+ qr/publish-target/,
+ 'publishing the fully reconciled target is an explicit final operation',
+);
my ($key_cmd, %key_opts);
$nvmet_mock->redefine(
--
2.54.0.windows.1
^ permalink raw reply related [flat|nested] 8+ messages in thread