[PATCH storage v2 2/7] zfs: add native NVMe/TCP storage backend

[Joaquin Varela <joaquinvarela@neatech.ar>]

Add the zfsnvme shared-storage type and an NVMET LUN provider.
ZFS continues to own volume lifecycle. Linux nvmet publishes
each zvol as a namespace, and native NVMe multipath combines
the configured paths.

Persist subsystem NQN, NSID and UUID as ZFS user properties.
Configfs is derived state and reconcile can rebuild it after a
target restart. Use the namespace UUID for the stable device
path exposed to QEMU.

Bind each connection to its configured host interface, require
native multipath, and support DH-HMAC-CHAP without exposing
secrets in argv. Add tests for parsing, identity, lifecycle and
connect behavior.

Signed-off-by: Joaquin Varela <joaquinvarela@neatech.ar>
---
 debian/control                   |   1 +
 src/PVE/Storage.pm               |   2 +
 src/PVE/Storage/LunCmd/Makefile  |   2 +-
 src/PVE/Storage/LunCmd/NVMET.pm  | 693 ++++++++++++++++++++++++++++++
 src/PVE/Storage/Makefile         |   1 +
 src/PVE/Storage/Plugin.pm        |   2 +-
 src/PVE/Storage/ZFSNVMePlugin.pm | 716 +++++++++++++++++++++++++++++++
 src/test/run_plugin_tests.pl     |   1 +
 src/test/zfsnvme_test.pm         | 350 +++++++++++++++
 9 files changed, 1766 insertions(+), 2 deletions(-)
 create mode 100644 src/PVE/Storage/LunCmd/NVMET.pm
 create mode 100644 src/PVE/Storage/ZFSNVMePlugin.pm
 create mode 100644 src/test/zfsnvme_test.pm

diff --git a/debian/control b/debian/control
index 3ad25e7..ac770a4 100644
--- a/debian/control
+++ b/debian/control
@@ -43,6 +43,7 @@ Depends: bzip2,
          lvm2,
          lzop,
          nfs-common,
+         nvme-cli,
          proxmox-backup-client (>= 2.1.10~),
          proxmox-backup-file-restore,
          pve-cluster (>= 5.0-32),
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 64ea9da..4356d83 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -36,6 +36,7 @@ use PVE::Storage::CephFSPlugin;
 use PVE::Storage::ISCSIDirectPlugin;
 use PVE::Storage::ZFSPoolPlugin;
 use PVE::Storage::ZFSPlugin;
+use PVE::Storage::ZFSNVMePlugin;
 use PVE::Storage::PBSPlugin;
 use PVE::Storage::BTRFSPlugin;
 use PVE::Storage::ESXiPlugin;
@@ -61,6 +62,7 @@ PVE::Storage::CephFSPlugin->register();
 PVE::Storage::ISCSIDirectPlugin->register();
 PVE::Storage::ZFSPoolPlugin->register();
 PVE::Storage::ZFSPlugin->register();
+PVE::Storage::ZFSNVMePlugin->register();
 PVE::Storage::PBSPlugin->register();
 PVE::Storage::BTRFSPlugin->register();
 PVE::Storage::ESXiPlugin->register();
diff --git a/src/PVE/Storage/LunCmd/Makefile b/src/PVE/Storage/LunCmd/Makefile
index a7209d1..2d30cad 100644
--- a/src/PVE/Storage/LunCmd/Makefile
+++ b/src/PVE/Storage/LunCmd/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Comstar.pm Istgt.pm Iet.pm LIO.pm
+SOURCES=Comstar.pm Istgt.pm Iet.pm LIO.pm NVMET.pm
 
 .PHONY: install
 install:
diff --git a/src/PVE/Storage/LunCmd/NVMET.pm b/src/PVE/Storage/LunCmd/NVMET.pm
new file mode 100644
index 0000000..7a902da
--- /dev/null
+++ b/src/PVE/Storage/LunCmd/NVMET.pm
@@ -0,0 +1,693 @@
+package PVE::Storage::LunCmd::NVMET;
+
+use v5.36;
+
+use PVE::Tools qw(run_command trim);
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+my $id_rsa_path = '/etc/pve/priv/zfs';
+
+my $REMOTE_HELPER = <<'REMOTE_HELPER';
+set -euo pipefail
+
+ROOT=/sys/kernel/config/nvmet
+MODEL='Proxmox ZFS NVMe'
+PROP_SUBSYS='proxmox:nvme-subsys'
+PROP_NSID='proxmox:nvme-nsid'
+PROP_UUID='proxmox:nvme-uuid'
+
+die() {
+    echo "$*" >&2
+    exit 1
+}
+
+validate_nqn() {
+    local value="$1"
+    [[ "$value" =~ ^nqn\.[A-Za-z0-9][A-Za-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._:-]*$ ]] ||
+        die "invalid NQN"
+    ((${#value} <= 223)) || die "NQN is too long"
+}
+
+validate_pool() {
+    [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.:/-]*$ ]] || die "invalid ZFS pool name"
+}
+
+validate_device() {
+    [[ "$1" =~ ^/dev/zvol/[A-Za-z0-9][A-Za-z0-9_.:/-]*$ ]] ||
+        die "invalid zvol device path"
+}
+
+validate_uuid() {
+    [[ "$1" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] ||
+        die "invalid namespace UUID"
+}
+
+prepare_configfs() {
+    modprobe nvmet_tcp
+    if ! mountpoint -q /sys/kernel/config; then
+        mount -t configfs none /sys/kernel/config
+    fi
+    [[ -d "$ROOT/subsystems" && -d "$ROOT/ports" && -d "$ROOT/hosts" ]] ||
+        die "NVMe target configfs is unavailable"
+
+    install -d -m 0755 /run/lock
+    exec 9>/run/lock/pve-nvmet.lock
+    flock -w 30 9 || die "timed out waiting for NVMe target configuration lock"
+}
+
+ensure_subsystem() {
+    local nqn="$1"
+    local subsystem="$ROOT/subsystems/$nqn"
+    local serial
+
+    validate_nqn "$nqn"
+    serial="PVEZFS$(printf '%s' "$nqn" | sha256sum | cut -c1-14)"
+
+    if [[ ! -d "$subsystem" ]]; then
+        mkdir "$subsystem"
+        printf '%s\n' "$MODEL" >"$subsystem/attr_model"
+        printf '%s\n' "$serial" >"$subsystem/attr_serial"
+    else
+        local current_model current_serial
+        current_model="$(<"$subsystem/attr_model")"
+        current_serial="$(<"$subsystem/attr_serial")"
+        if [[ "$current_model" != "$MODEL" || "$current_serial" != "$serial" ]]; then
+            if find "$subsystem/namespaces" -mindepth 1 -maxdepth 1 -type d -print -quit |
+                grep -q .; then
+                die "refusing to take over existing NVMe subsystem '$nqn'"
+            fi
+            printf '%s\n' "$MODEL" >"$subsystem/attr_model"
+            printf '%s\n' "$serial" >"$subsystem/attr_serial"
+        fi
+    fi
+
+    printf '0\n' >"$subsystem/attr_allow_any_host"
+}
+
+find_port() {
+    local family="$1"
+    local address="$2"
+    local service="$3"
+    local port
+
+    for port in "$ROOT"/ports/*; do
+        [[ -d "$port" ]] || continue
+        [[ "$(<"$port/addr_trtype")" == tcp ]] || continue
+        [[ "$(<"$port/addr_adrfam")" == "$family" ]] || continue
+        [[ "$(<"$port/addr_traddr")" == "$address" ]] || continue
+        [[ "$(<"$port/addr_trsvcid")" == "$service" ]] || continue
+        basename "$port"
+        return 0
+    done
+    return 1
+}
+
+allocate_port_id() {
+    local id=1
+    while [[ -e "$ROOT/ports/$id" ]]; do
+        ((id++))
+    done
+    printf '%s\n' "$id"
+}
+
+ensure_port() {
+    local nqn="$1"
+    local spec="$2"
+    local family address service id port
+
+    IFS=, read -r family address service <<<"$spec"
+    [[ "$family" == ipv4 || "$family" == ipv6 ]] || die "invalid address family"
+    [[ "$service" =~ ^[0-9]{1,5}$ ]] || die "invalid NVMe/TCP service"
+    ((service >= 1 && service <= 65535)) || die "invalid NVMe/TCP service"
+    if [[ "$family" == ipv4 ]]; then
+        [[ "$address" =~ ^[0-9.]+$ ]] || die "invalid IPv4 target address"
+    else
+        [[ "$address" =~ ^[0-9a-fA-F:]+$ ]] || die "invalid IPv6 target address"
+    fi
+
+    id="$(find_port "$family" "$address" "$service" || true)"
+    if [[ -z "$id" ]]; then
+        id="$(allocate_port_id)"
+        port="$ROOT/ports/$id"
+        mkdir "$port"
+        printf 'tcp\n' >"$port/addr_trtype"
+        printf '%s\n' "$family" >"$port/addr_adrfam"
+        printf '%s\n' "$address" >"$port/addr_traddr"
+        printf '%s\n' "$service" >"$port/addr_trsvcid"
+    fi
+
+    port="$ROOT/ports/$id"
+    if [[ ! -e "$port/subsystems/$nqn" ]]; then
+        ln -s "$ROOT/subsystems/$nqn" "$port/subsystems/$nqn"
+    fi
+    printf '%s\n' "$id"
+}
+
+ensure_target() {
+    local nqn="$1"
+    local pool="$2"
+    shift 2
+    local spec id port desired_ports=''
+
+    validate_nqn "$nqn"
+    validate_pool "$pool"
+    (($# >= 1)) || die "at least one NVMe/TCP portal is required"
+    ensure_subsystem "$nqn"
+
+    for spec in "$@"; do
+        id="$(ensure_port "$nqn" "$spec")"
+        desired_ports="$desired_ports $id"
+    done
+
+    for port in "$ROOT"/ports/*; do
+        [[ -d "$port" && -L "$port/subsystems/$nqn" ]] || continue
+        id="$(basename "$port")"
+        case " $desired_ports " in
+            *" $id "*) ;;
+            *) rm "$port/subsystems/$nqn" ;;
+        esac
+    done
+}
+
+ensure_host() {
+    local nqn="$1"
+    local hostnqn="$2"
+    validate_nqn "$nqn"
+    validate_nqn "$hostnqn"
+    [[ -d "$ROOT/subsystems/$nqn" ]] || die "NVMe subsystem does not exist"
+    [[ -d "$ROOT/hosts/$hostnqn" ]] || mkdir "$ROOT/hosts/$hostnqn"
+}
+
+allow_host() {
+    local nqn="$1"
+    local hostnqn="$2"
+    validate_nqn "$nqn"
+    validate_nqn "$hostnqn"
+    [[ -d "$ROOT/hosts/$hostnqn" ]] || die "NVMe host does not exist"
+    if [[ ! -e "$ROOT/subsystems/$nqn/allowed_hosts/$hostnqn" ]]; then
+        ln -s "$ROOT/hosts/$hostnqn" "$ROOT/subsystems/$nqn/allowed_hosts/$hostnqn"
+    fi
+}
+
+list_volumes() {
+    local pool="$1"
+    local dataset property value source
+    declare -A datasets=()
+    declare -A subsystems=()
+    declare -A nsids=()
+    declare -A uuids=()
+
+    while IFS=$'\t' read -r dataset property value source; do
+        datasets["$dataset"]=1
+        if [[ "$source" != local && "$source" != received ]]; then
+            value='-'
+        fi
+        case "$property" in
+            "$PROP_SUBSYS") subsystems["$dataset"]="$value" ;;
+            "$PROP_NSID") nsids["$dataset"]="$value" ;;
+            "$PROP_UUID") uuids["$dataset"]="$value" ;;
+        esac
+    done < <(
+        zfs get -H -r -t volume -o name,property,value,source \
+            "$PROP_SUBSYS,$PROP_NSID,$PROP_UUID" "$pool"
+    )
+
+    for dataset in "${!datasets[@]}"; do
+        printf '%s\t%s\t%s\t%s\n' \
+            "$dataset" \
+            "${subsystems[$dataset]:--}" \
+            "${nsids[$dataset]:--}" \
+            "${uuids[$dataset]:--}"
+    done
+}
+
+property_value() {
+    local property="$1"
+    local dataset="$2"
+    local output value source
+    output="$(zfs get -H -o value,source "$property" "$dataset" 2>/dev/null || true)"
+    IFS=$'\t' read -r value source <<<"$output"
+    if [[ -z "$value" || ("$source" != local && "$source" != received) ]]; then
+        value='-'
+    fi
+    printf '%s\n' "$value"
+}
+
+identity_is_duplicate() {
+    local pool="$1"
+    local dataset="$2"
+    local nqn="$3"
+    local nsid="$4"
+    local uuid="$5"
+    local other other_nqn other_nsid other_uuid
+
+    while IFS=$'\t' read -r other other_nqn other_nsid other_uuid; do
+        [[ "$other" != "$dataset" && "$other_nqn" == "$nqn" ]] || continue
+        [[ "$other_nsid" == "$nsid" || "$other_uuid" == "$uuid" ]] && return 0
+    done < <(list_volumes "$pool")
+    return 1
+}
+
+allocate_nsid() {
+    local pool="$1"
+    local nqn="$2"
+    local dataset other_nqn other_nsid other_uuid namespace nsid=1
+    declare -A used=()
+
+    while IFS=$'\t' read -r dataset other_nqn other_nsid other_uuid; do
+        if [[ "$other_nqn" == "$nqn" && "$other_nsid" =~ ^[1-9][0-9]*$ ]]; then
+            used["$other_nsid"]=1
+        fi
+    done < <(list_volumes "$pool")
+
+    for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+        [[ -d "$namespace" ]] || continue
+        used["$(basename "$namespace")"]=1
+    done
+
+    while [[ -n "${used[$nsid]:-}" ]]; do
+        ((nsid++))
+    done
+    printf '%s\n' "$nsid"
+}
+
+ensure_identity() {
+    local nqn="$1"
+    local pool="$2"
+    local dataset="$3"
+    local current_nqn nsid uuid
+
+    current_nqn="$(property_value "$PROP_SUBSYS" "$dataset")"
+    nsid="$(property_value "$PROP_NSID" "$dataset")"
+    uuid="$(property_value "$PROP_UUID" "$dataset")"
+
+    if [[ "$current_nqn" != '-' && "$current_nqn" != "$nqn" ]]; then
+        die "ZFS volume '$dataset' belongs to NVMe subsystem '$current_nqn'"
+    fi
+
+    if [[ ! "$nsid" =~ ^[1-9][0-9]*$ ]] ||
+        [[ ! "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]] ||
+        identity_is_duplicate "$pool" "$dataset" "$nqn" "$nsid" "$uuid"; then
+        nsid="$(allocate_nsid "$pool" "$nqn")"
+        uuid="$(< /proc/sys/kernel/random/uuid)"
+    fi
+
+    validate_uuid "$uuid"
+    zfs set "$PROP_SUBSYS=$nqn" "$PROP_NSID=$nsid" "$PROP_UUID=$uuid" "$dataset"
+    IDENTITY_NSID="$nsid"
+    IDENTITY_UUID="$uuid"
+}
+
+remove_namespace() {
+    local namespace="$1"
+    if [[ "$(<"$namespace/enable")" == 1 ]]; then
+        printf '0\n' >"$namespace/enable"
+    fi
+    rmdir "$namespace"
+}
+
+ensure_namespace() {
+    local nqn="$1"
+    local nsid="$2"
+    local uuid="$3"
+    local device="$4"
+    local namespace="$ROOT/subsystems/$nqn/namespaces/$nsid"
+    local other
+
+    validate_uuid "$uuid"
+    validate_device "$device"
+    [[ -b "$device" ]] || die "zvol '$device' is not a block device"
+    [[ -d "$ROOT/subsystems/$nqn" ]] || die "NVMe subsystem does not exist"
+
+    for other in "$ROOT/subsystems/$nqn"/namespaces/*; do
+        [[ -d "$other" && "$other" != "$namespace" ]] || continue
+        if [[ "$(<"$other/device_uuid")" == "$uuid" ]]; then
+            die "namespace UUID '$uuid' is already in use"
+        fi
+    done
+
+    if [[ -d "$namespace" ]]; then
+        if [[ "$(<"$namespace/device_uuid")" == "$uuid" ]] &&
+            [[ "$(<"$namespace/device_path")" == "$device" ]]; then
+            [[ "$(<"$namespace/enable")" == 1 ]] || printf '1\n' >"$namespace/enable"
+            return
+        fi
+        remove_namespace "$namespace"
+    fi
+
+    mkdir "$namespace"
+    if ! {
+        printf '%s\n' "$device" >"$namespace/device_path"
+        printf '%s\n' "$uuid" >"$namespace/device_uuid"
+        printf '0\n' >"$namespace/buffered_io"
+        printf '1\n' >"$namespace/enable"
+    }; then
+        printf '0\n' >"$namespace/enable" 2>/dev/null || true
+        rmdir "$namespace" 2>/dev/null || true
+        die "failed to create namespace '$nsid'"
+    fi
+}
+
+create_volume() {
+    local nqn="$1"
+    local pool="$2"
+    local device="$3"
+    local dataset
+
+    validate_nqn "$nqn"
+    validate_pool "$pool"
+    validate_device "$device"
+    dataset="${device#/dev/zvol/}"
+    [[ "$dataset" == "$pool/"* ]] || die "zvol is outside configured pool"
+    [[ "$(zfs get -H -o value type "$dataset")" == volume ]] ||
+        die "'$dataset' is not a ZFS volume"
+
+    ensure_identity "$nqn" "$pool" "$dataset"
+    ensure_namespace "$nqn" "$IDENTITY_NSID" "$IDENTITY_UUID" "$device"
+    printf '%s\n' "$IDENTITY_UUID"
+}
+
+delete_volume() {
+    local nqn="$1"
+    local uuid="$2"
+    local namespace found=0
+
+    validate_nqn "$nqn"
+    validate_uuid "$uuid"
+    [[ -d "$ROOT/subsystems/$nqn" ]] || return 0
+    for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+        [[ -d "$namespace" ]] || continue
+        [[ "$(<"$namespace/device_uuid")" == "$uuid" ]] || continue
+        ((found == 0)) || die "duplicate namespace UUID '$uuid'"
+        remove_namespace "$namespace"
+        found=1
+    done
+}
+
+lookup_volume() {
+    local nqn="$1"
+    local pool="$2"
+    local device="$3"
+    local dataset current_nqn nsid uuid
+
+    validate_nqn "$nqn"
+    validate_pool "$pool"
+    validate_device "$device"
+    dataset="${device#/dev/zvol/}"
+    [[ "$dataset" == "$pool/"* ]] || die "zvol is outside configured pool"
+    [[ "$(zfs get -H -o value type "$dataset")" == volume ]] ||
+        die "'$dataset' is not a ZFS volume"
+
+    current_nqn="$(property_value "$PROP_SUBSYS" "$dataset")"
+    nsid="$(property_value "$PROP_NSID" "$dataset")"
+    uuid="$(property_value "$PROP_UUID" "$dataset")"
+    [[ "$current_nqn" == "$nqn" ]] ||
+        die "ZFS volume '$dataset' is not owned by NVMe subsystem '$nqn'"
+    [[ "$nsid" =~ ^[1-9][0-9]*$ ]] || die "invalid NSID on '$dataset'"
+    validate_uuid "$uuid"
+    identity_is_duplicate "$pool" "$dataset" "$nqn" "$nsid" "$uuid" &&
+        die "duplicate NVMe identity on '$dataset'"
+
+    ensure_namespace "$nqn" "$nsid" "$uuid" "$device"
+    printf '%s\n' "$uuid"
+}
+
+lookup_view() {
+    local nqn="$1"
+    local pool="$2"
+    local uuid="$3"
+    local dataset other_nqn nsid other_uuid found=''
+
+    validate_nqn "$nqn"
+    validate_pool "$pool"
+    validate_uuid "$uuid"
+    while IFS=$'\t' read -r dataset other_nqn nsid other_uuid; do
+        [[ "$other_nqn" == "$nqn" && "$other_uuid" == "$uuid" ]] || continue
+        [[ -z "$found" ]] || die "duplicate namespace UUID '$uuid'"
+        found="$nsid"
+    done < <(list_volumes "$pool")
+    [[ "$found" =~ ^[1-9][0-9]*$ ]] || die "namespace UUID '$uuid' was not found"
+    printf '%s\n' "$found"
+}
+
+resize_volume() {
+    local nqn="$1"
+    local uuid="$2"
+    local namespace
+
+    validate_nqn "$nqn"
+    validate_uuid "$uuid"
+    for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+        [[ -d "$namespace" ]] || continue
+        if [[ "$(<"$namespace/device_uuid")" == "$uuid" ]]; then
+            printf '1\n' >"$namespace/revalidate_size"
+            return
+        fi
+    done
+    die "namespace UUID '$uuid' was not found"
+}
+
+reconcile() {
+    local nqn="$1"
+    local pool="$2"
+    local dataset other_nqn nsid uuid device namespace
+    declare -A desired=()
+    declare -A seen_uuid=()
+
+    validate_nqn "$nqn"
+    validate_pool "$pool"
+    [[ -d "$ROOT/subsystems/$nqn" ]] || die "NVMe subsystem does not exist"
+
+    while IFS=$'\t' read -r dataset other_nqn nsid uuid; do
+        [[ "$other_nqn" == "$nqn" ]] || continue
+        [[ "$nsid" =~ ^[1-9][0-9]*$ ]] || die "invalid NSID on '$dataset'"
+        validate_uuid "$uuid"
+        [[ -z "${desired[$nsid]:-}" ]] || die "duplicate NSID '$nsid'"
+        [[ -z "${seen_uuid[$uuid]:-}" ]] || die "duplicate namespace UUID '$uuid'"
+        desired["$nsid"]=1
+        seen_uuid["$uuid"]=1
+        device="/dev/zvol/$dataset"
+        ensure_namespace "$nqn" "$nsid" "$uuid" "$device"
+    done < <(list_volumes "$pool")
+
+    for namespace in "$ROOT/subsystems/$nqn"/namespaces/*; do
+        [[ -d "$namespace" ]] || continue
+        nsid="$(basename "$namespace")"
+        [[ -n "${desired[$nsid]:-}" ]] || remove_namespace "$namespace"
+    done
+}
+
+delete_target() {
+    local nqn="$1"
+    local subsystem="$ROOT/subsystems/$nqn"
+    local namespace port link hostnqn referenced
+    local -a hosts=()
+
+    validate_nqn "$nqn"
+    [[ -d "$subsystem" ]] || return 0
+
+    for namespace in "$subsystem"/namespaces/*; do
+        [[ -d "$namespace" ]] || continue
+        remove_namespace "$namespace"
+    done
+    for port in "$ROOT"/ports/*; do
+        [[ -d "$port" ]] || continue
+        link="$port/subsystems/$nqn"
+        [[ -L "$link" ]] && rm "$link"
+    done
+    for link in "$subsystem"/allowed_hosts/*; do
+        [[ -L "$link" ]] || continue
+        hosts+=("$(basename "$link")")
+        rm "$link"
+    done
+    rmdir "$subsystem"
+
+    # Host objects (and their DHCHAP keys) are global in nvmet. Remove an
+    # orphan only after proving that no other subsystem still authorizes it.
+    for hostnqn in "${hosts[@]}"; do
+        referenced=0
+        for link in "$ROOT"/subsystems/*/allowed_hosts/"$hostnqn"; do
+            if [[ -L "$link" ]]; then
+                referenced=1
+                break
+            fi
+        done
+        ((referenced == 1)) || rmdir "$ROOT/hosts/$hostnqn"
+    done
+}
+
+prepare_configfs
+mode="${1:-}"
+shift || true
+
+case "$mode" in
+    ensure-target) ensure_target "$@" ;;
+    ensure-host) ensure_host "$@" ;;
+    allow-host) allow_host "$@" ;;
+    create) create_volume "$@" ;;
+    delete) delete_volume "$@" ;;
+    lookup) lookup_volume "$@" ;;
+    view) lookup_view "$@" ;;
+    resize) resize_volume "$@" ;;
+    reconcile) reconcile "$@" ;;
+    delete-target) delete_target "$@" ;;
+    *) die "unknown NVMe target operation" ;;
+esac
+REMOTE_HELPER
+
+my $REMOTE_SET_HOST_KEY = <<'REMOTE_SET_HOST_KEY';
+set -euo pipefail
+
+ROOT=/sys/kernel/config/nvmet
+hostnqn="$1"
+host="$ROOT/hosts/$hostnqn"
+
+[[ "$hostnqn" =~ ^nqn\.[A-Za-z0-9][A-Za-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._:-]*$ ]] || {
+    echo "invalid host NQN '$hostnqn'" >&2
+    exit 1
+}
+((${#hostnqn} <= 223)) || {
+    echo 'host NQN is too long' >&2
+    exit 1
+}
+[[ -d "$host" ]] || {
+    echo 'NVMe host does not exist' >&2
+    exit 1
+}
+
+IFS= read -r key
+[[ "$key" =~ ^DHHC-1:[0-9A-Fa-f]{2}:[A-Za-z0-9+/=]+:$ ]] || {
+    echo 'invalid DH-HMAC-CHAP key representation' >&2
+    exit 1
+}
+current="$(<"$host/dhchap_key")"
+if [[ -n "$current" && "$current" != "$key" ]]; then
+    for link in "$ROOT"/subsystems/*/allowed_hosts/"$hostnqn"; do
+        if [[ -L "$link" ]]; then
+            echo "refusing to replace an in-use DH-HMAC-CHAP key for '$hostnqn'" >&2
+            exit 1
+        fi
+    done
+fi
+printf '%s\n' "$key" >"$host/dhchap_key"
+REMOTE_SET_HOST_KEY
+
+my sub server($scfg) {
+    return $scfg->{server} // $scfg->{portal};
+}
+
+my sub ssh_key($scfg) {
+    my $server = server($scfg);
+    return "$id_rsa_path/${server}_id_rsa";
+}
+
+my sub remote_call($scfg, $timeout, $operation, @params) {
+    my $server = server($scfg);
+    my $target = 'root@' . $server;
+    my $msg = '';
+    my $err = '';
+    my $cmd = [
+        @ssh_cmd,
+        '-i',
+        ssh_key($scfg),
+        $target,
+        '--',
+        '/bin/bash',
+        '-s',
+        '--',
+        $operation,
+        @params,
+    ];
+
+    run_command(
+        $cmd,
+        input => $REMOTE_HELPER,
+        timeout => $timeout // 10,
+        outfunc => sub($line) { $msg .= "$line\n" },
+        errfunc => sub($line) { $err .= "$line\n" },
+        errmsg => "NVMe target operation '$operation' failed",
+    );
+
+    return trim($msg);
+}
+
+sub get_base($scfg) {
+    return '/dev/zvol';
+}
+
+sub ensure_target($scfg, $hostnqn, $portals) {
+    my $nqn = $scfg->{subsysnqn};
+
+    remote_call($scfg, 15, 'ensure-target', $nqn, $scfg->{pool}, $portals->@*);
+    remote_call($scfg, 10, 'ensure-host', $nqn, $hostnqn);
+}
+
+sub set_host_key($scfg, $hostnqn, $key) {
+    my $server = server($scfg);
+    my $target = 'root@' . $server;
+    my $remote_cmd = join(
+        ' ',
+        map { PVE::Tools::shellquote($_) }
+            ('/bin/bash', '-c', $REMOTE_SET_HOST_KEY, '--', $hostnqn),
+    );
+    my $cmd = [
+        @ssh_cmd,
+        '-i',
+        ssh_key($scfg),
+        $target,
+        '--',
+        $remote_cmd,
+    ];
+
+    run_command(
+        $cmd,
+        input => "$key\n",
+        timeout => 10,
+        quiet => 1,
+        errmsg => "failed to configure NVMe DH-HMAC-CHAP key",
+    );
+}
+
+sub allow_host($scfg, $hostnqn) {
+    remote_call($scfg, 10, 'allow-host', $scfg->{subsysnqn}, $hostnqn);
+}
+
+sub reconcile($scfg) {
+    remote_call($scfg, 30, 'reconcile', $scfg->{subsysnqn}, $scfg->{pool});
+}
+
+sub delete_target($scfg) {
+    remote_call($scfg, 15, 'delete-target', $scfg->{subsysnqn});
+}
+
+my %lun_cmd_map = (
+    create_lu => sub($scfg, $timeout, $method, $device) {
+        return remote_call($scfg, $timeout, 'create', $scfg->{subsysnqn}, $scfg->{pool}, $device);
+    },
+    delete_lu => sub($scfg, $timeout, $method, $uuid) {
+        return remote_call($scfg, $timeout, 'delete', $scfg->{subsysnqn}, $uuid);
+    },
+    import_lu => sub($scfg, $timeout, $method, $device) {
+        return remote_call($scfg, $timeout, 'create', $scfg->{subsysnqn}, $scfg->{pool}, $device);
+    },
+    modify_lu => sub($scfg, $timeout, $method, $size, $uuid) {
+        return remote_call($scfg, $timeout, 'resize', $scfg->{subsysnqn}, $uuid);
+    },
+    add_view => sub($scfg, $timeout, $method, @params) {
+        return '';
+    },
+    list_view => sub($scfg, $timeout, $method, $uuid) {
+        return remote_call($scfg, $timeout, 'view', $scfg->{subsysnqn}, $scfg->{pool}, $uuid);
+    },
+    list_lu => sub($scfg, $timeout, $method, $device) {
+        return remote_call($scfg, $timeout, 'lookup', $scfg->{subsysnqn}, $scfg->{pool}, $device);
+    },
+);
+
+sub run_lun_command($scfg, $timeout, $method, @params) {
+    die "unknown command '$method'\n" if !exists($lun_cmd_map{$method});
+    return $lun_cmd_map{$method}->($scfg, $timeout, $method, @params);
+}
+
+1;
diff --git a/src/PVE/Storage/Makefile b/src/PVE/Storage/Makefile
index a67dc25..d1cbfe2 100644
--- a/src/PVE/Storage/Makefile
+++ b/src/PVE/Storage/Makefile
@@ -11,6 +11,7 @@ SOURCES= \
 	ISCSIDirectPlugin.pm \
 	ZFSPoolPlugin.pm \
 	ZFSPlugin.pm \
+	ZFSNVMePlugin.pm \
 	PBSPlugin.pm \
 	BTRFSPlugin.pm \
 	LvmThinPlugin.pm \
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 4f69f9b..3a45a73 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -35,7 +35,7 @@ our @COMMON_TAR_FLAGS = qw(
 );
 
 our @SHARED_STORAGE = (
-    'iscsi', 'nfs', 'cifs', 'rbd', 'cephfs', 'iscsidirect', 'zfs', 'drbd', 'pbs',
+    'iscsi', 'nfs', 'cifs', 'rbd', 'cephfs', 'iscsidirect', 'zfs', 'zfsnvme', 'drbd', 'pbs',
 );
 
 our $QCOW2_PREALLOCATION = {
diff --git a/src/PVE/Storage/ZFSNVMePlugin.pm b/src/PVE/Storage/ZFSNVMePlugin.pm
new file mode 100644
index 0000000..83d086c
--- /dev/null
+++ b/src/PVE/Storage/ZFSNVMePlugin.pm
@@ -0,0 +1,716 @@
+package PVE::Storage::ZFSNVMePlugin;
+
+use v5.36;
+
+use File::Path qw(make_path);
+use IO::Socket::IP;
+use IO::File;
+use JSON;
+
+use PVE::JSONSchema;
+use PVE::RESTEnvironment qw(log_warn);
+use PVE::Storage::LunCmd::NVMET;
+use PVE::Storage::ZFSPlugin;
+use PVE::Tools qw(file_read_firstline file_set_contents run_command trim);
+
+use base qw(PVE::Storage::ZFSPlugin);
+
+my $nvme = '/usr/sbin/nvme';
+my $secret_dir = '/etc/pve/priv/storage';
+my $runtime_dir = '/run/pve-storage';
+my $max_paths = 16;
+
+my $RE_NQN = qr{
+    \A
+    nqn \.
+    [A-Za-z0-9] [A-Za-z0-9.-]*
+    :
+    [A-Za-z0-9] [A-Za-z0-9._:-]*
+    \z
+}nxx;
+my $RE_IPV4_PORTAL = qr{
+    \A
+    (?<address> [^:]+)
+    (?: : (?<port> [0-9]+))?
+    \z
+}nxx;
+my $RE_IPV6_PORTAL = qr{
+    \A
+    \[ (?<address> [^\]]+) \]
+    (?: : (?<port> [0-9]+))?
+    \z
+}nxx;
+my $RE_HOST_IFACE = qr{\A [A-Za-z0-9_.-]+ \z}nxx;
+my $RE_DHCHAP_KEY = qr{
+    \A DHHC-1 : [0-9A-Fa-f]{2} : [A-Za-z0-9+/=]+ : \z
+}nxx;
+my $RE_NVME_CONTROLLER = qr{\A nvme [0-9]+ \z}nxx;
+my $RE_NVME_SUBSYSTEM = qr{\A nvme-subsys [0-9]+ \z}nxx;
+my $RE_TRADDR = qr{(?: \A | ,) traddr=(?<value>[^,]+)}nxx;
+my $RE_TRSVCID = qr{(?: \A | ,) trsvcid=(?<value>[^,]+)}nxx;
+my $RE_HOST_IFACE_ADDRESS = qr{(?: \A | ,) host_iface=(?<value>[^,]+)}nxx;
+
+sub verify_nvme_nqn($value, $noerr = undef) {
+
+    if (
+        length($value) > 223
+        || $value !~ $RE_NQN
+    ) {
+        return undef if $noerr;
+        die "value is not a valid NVMe qualified name\n";
+    }
+
+    return $value;
+}
+
+sub parse_nvme_portals($value, $noerr = undef) {
+    my $result = [];
+    my $seen = {};
+
+    for my $entry (split(/,/, $value // '')) {
+        $entry = trim($entry);
+        my ($address, $port, $family);
+        if ($entry =~ $RE_IPV6_PORTAL) {
+            ($address, $port, $family) = ($+{address}, $+{port} // 4420, 'ipv6');
+        } elsif ($entry =~ $RE_IPV4_PORTAL) {
+            ($address, $port, $family) = ($+{address}, $+{port} // 4420, 'ipv4');
+        } else {
+            return undef if $noerr;
+            die "invalid NVMe/TCP portal '$entry'\n";
+        }
+
+        if (
+            !PVE::JSONSchema::pve_verify_ip($address, 1)
+            || ($family eq 'ipv4' && index($address, ':') >= 0)
+            || ($family eq 'ipv6' && index($address, ':') < 0)
+            || $port < 1
+            || $port > 65535
+        ) {
+            return undef if $noerr;
+            die "invalid NVMe/TCP portal '$entry'\n";
+        }
+        my $id = join("\0", $family, $address, $port);
+        if ($seen->{$id}++) {
+            return undef if $noerr;
+            die "duplicate NVMe/TCP portal '$entry'\n";
+        }
+        push $result->@*, {
+            address => $address,
+            port => int($port),
+            family => $family,
+        };
+        if (scalar($result->@*) > $max_paths) {
+            return undef if $noerr;
+            die "at most $max_paths NVMe/TCP portals are supported\n";
+        }
+    }
+
+    if (!$result->@*) {
+        return undef if $noerr;
+        die "at least one NVMe/TCP portal is required\n";
+    }
+
+    return $result;
+}
+
+my sub verify_nvme_portals($value, $noerr = undef) {
+    return undef if !parse_nvme_portals($value, $noerr);
+    return $value;
+}
+
+sub parse_nvme_host_ifaces($value, $noerr = undef) {
+    my $result = [];
+
+    for my $iface (split(/,/, $value // '')) {
+        $iface = trim($iface);
+        if (length($iface) < 1 || length($iface) > 15 || $iface !~ $RE_HOST_IFACE) {
+            return undef if $noerr;
+            die "invalid NVMe/TCP host interface '$iface'\n";
+        }
+        push $result->@*, $iface;
+    }
+
+    if (!$result->@*) {
+        return undef if $noerr;
+        die "at least one NVMe/TCP host interface is required\n";
+    }
+
+    return $result;
+}
+
+my sub verify_nvme_host_ifaces($value, $noerr = undef) {
+    return undef if !parse_nvme_host_ifaces($value, $noerr);
+    return $value;
+}
+
+sub _configured_portals($scfg) {
+    my $portals = parse_nvme_portals($scfg->{'nvme-portals'});
+    my $ifaces = parse_nvme_host_ifaces($scfg->{'nvme-host-ifaces'});
+    die "nvme-host-ifaces must contain one interface for each nvme-portals entry\n"
+        if scalar($ifaces->@*) != scalar($portals->@*);
+
+    for (my $i = 0; $i < scalar($portals->@*); $i++) {
+        $portals->[$i]->{host_iface} = $ifaces->[$i];
+    }
+    return $portals;
+}
+
+PVE::JSONSchema::register_format('pve-storage-nvme-nqn', \&verify_nvme_nqn);
+PVE::JSONSchema::register_format('pve-storage-nvme-portals', \&verify_nvme_portals);
+PVE::JSONSchema::register_format('pve-storage-nvme-host-ifaces', \&verify_nvme_host_ifaces);
+
+sub type($class) {
+    return 'zfsnvme';
+}
+
+sub plugindata($class) {
+    return {
+        content => [{ images => 1 }, { images => 1 }],
+        'sensitive-properties' => { 'dhchap-key' => 1 },
+    };
+}
+
+sub properties($class) {
+    return {
+        subsysnqn => {
+            description => "NVMe subsystem qualified name.",
+            type => 'string',
+            format => 'pve-storage-nvme-nqn',
+        },
+        'nvme-portals' => {
+            description =>
+                "Comma-separated NVMe/TCP target addresses. The default TCP port is 4420.",
+            type => 'string',
+            format => 'pve-storage-nvme-portals',
+            maxLength => 2048,
+        },
+        'nvme-host-ifaces' => {
+            description =>
+                "Comma-separated local interfaces, in portal order. Interface names must be identical on every cluster node.",
+            type => 'string',
+            format => 'pve-storage-nvme-host-ifaces',
+            maxLength => 512,
+        },
+        'dhchap-key' => {
+            description => "NVMe DH-HMAC-CHAP key in secret representation format.",
+            type => 'string',
+            maxLength => 256,
+        },
+        'nvme-iopolicy' => {
+            description => "Native NVMe multipath I/O policy.",
+            type => 'string',
+            enum => ['numa', 'round-robin', 'queue-depth'],
+            default => 'round-robin',
+        },
+        'nvme-keep-alive-tmo' => {
+            description => "NVMe keep-alive timeout in seconds.",
+            type => 'integer',
+            minimum => 1,
+            maximum => 120,
+            default => 5,
+        },
+        'nvme-reconnect-delay' => {
+            description => "Delay between NVMe reconnect attempts in seconds.",
+            type => 'integer',
+            minimum => 1,
+            maximum => 120,
+            default => 2,
+        },
+        'nvme-ctrl-loss-tmo' => {
+            description => "Time to keep retrying a lost NVMe controller in seconds.",
+            type => 'integer',
+            minimum => -1,
+            maximum => 86400,
+            default => 600,
+        },
+        'nvme-nr-io-queues' => {
+            description => "Number of NVMe/TCP I/O queues per controller.",
+            type => 'integer',
+            minimum => 1,
+            maximum => 1024,
+            optional => 1,
+        },
+    };
+}
+
+sub options($class) {
+    return {
+        server => { fixed => 1 },
+        subsysnqn => { fixed => 1 },
+        'nvme-portals' => { fixed => 1 },
+        'nvme-host-ifaces' => { optional => 1 },
+        pool => { fixed => 1 },
+        blocksize => { fixed => 1 },
+        sparse => { optional => 1 },
+        'dhchap-key' => { optional => 1 },
+        'nvme-iopolicy' => { optional => 1 },
+        'nvme-keep-alive-tmo' => { optional => 1 },
+        'nvme-reconnect-delay' => { optional => 1 },
+        'nvme-ctrl-loss-tmo' => { optional => 1 },
+        'nvme-nr-io-queues' => { optional => 1 },
+        nodes => { optional => 1 },
+        disable => { optional => 1 },
+        content => { optional => 1 },
+        bwlimit => { optional => 1 },
+    };
+}
+
+sub check_config($class, $section_id, $config, $create, $skip_schema_check) {
+    if ($create) {
+        $config->{sparse} = 1 if !defined($config->{sparse});
+        $config->{'nvme-iopolicy'} //= 'round-robin';
+        $config->{'nvme-keep-alive-tmo'} //= 5;
+        $config->{'nvme-reconnect-delay'} //= 2;
+        $config->{'nvme-ctrl-loss-tmo'} //= 600;
+    }
+
+    verify_nvme_nqn($config->{subsysnqn}) if defined($config->{subsysnqn});
+    parse_nvme_portals($config->{'nvme-portals'}) if defined($config->{'nvme-portals'});
+    if (defined($config->{'nvme-host-ifaces'}) && defined($config->{'nvme-portals'})) {
+        _configured_portals($config);
+    } elsif (defined($config->{'nvme-host-ifaces'})) {
+        parse_nvme_host_ifaces($config->{'nvme-host-ifaces'});
+    }
+    return $class->SUPER::check_config($section_id, $config, $create, $skip_schema_check);
+}
+
+sub zfs_lun_provider($class, $scfg = undef) {
+    return 'PVE::Storage::LunCmd::NVMET';
+}
+
+sub zfs_request($class, $scfg, @params) {
+    local $scfg->{portal} = $scfg->{server};
+    return $class->SUPER::zfs_request($scfg, @params);
+}
+
+sub zfs_list_zvol($class, $scfg) {
+    my $list = $class->SUPER::zfs_list_zvol($scfg);
+    my $properties = $class->zfs_request(
+        $scfg,
+        10,
+        'get',
+        '-H',
+        '-d',
+        '1',
+        '-o',
+        'name,value,source',
+        'proxmox:nvme-subsys',
+        $scfg->{pool},
+    );
+    my $owned = {};
+    for my $line (split(/\n/, $properties)) {
+        my ($dataset, $nqn, $source) = split(/\t/, $line, 3);
+        next if !defined($source) || ($source ne 'local' && $source ne 'received');
+        next if !defined($nqn) || $nqn ne $scfg->{subsysnqn};
+        my $prefix = "$scfg->{pool}/";
+        next if index($dataset, $prefix) != 0;
+        $owned->{substr($dataset, length($prefix))} = 1;
+    }
+    for my $name (keys $list->%*) {
+        delete $list->{$name} if !$owned->{$name};
+    }
+    return $list;
+}
+
+my sub secret_path($storeid) {
+    return "$secret_dir/$storeid.nvme-dhchap";
+}
+
+sub _assert_unique_target($storeid, $scfg, $cfg = undef) {
+    $cfg //= PVE::Storage::config();
+    my $ids = $cfg->{ids} // {};
+    for my $other_id (keys $ids->%*) {
+        next if $other_id eq $storeid;
+        my $other = $cfg->{ids}->{$other_id};
+        next if ($other->{type} // '') ne 'zfsnvme';
+
+        die "NVMe subsystem NQN is already used by storage '$other_id'\n"
+            if ($other->{subsysnqn} // '') eq $scfg->{subsysnqn};
+        die "ZFS pool '$scfg->{pool}' on '$scfg->{server}' is already used by storage '$other_id'\n"
+            if ($other->{server} // '') eq $scfg->{server}
+            && ($other->{pool} // '') eq $scfg->{pool};
+    }
+}
+
+sub _validate_secret($key) {
+    die "missing NVMe DH-HMAC-CHAP key\n" if !defined($key) || $key eq '';
+    die "invalid NVMe DH-HMAC-CHAP key representation\n"
+        if $key !~ $RE_DHCHAP_KEY;
+    return $key;
+}
+
+my sub set_secret($storeid, $key) {
+    _validate_secret($key);
+    make_path($secret_dir, { mode => 0700 });
+    file_set_contents(secret_path($storeid), "$key\n", 0600);
+}
+
+my sub get_secret($storeid) {
+    my $key = file_read_firstline(secret_path($storeid));
+    return _validate_secret($key);
+}
+
+my sub delete_secret($storeid) {
+    unlink(secret_path($storeid));
+}
+
+sub on_add_hook($class, $storeid, $scfg, %sensitive) {
+    _configured_portals($scfg);
+    _assert_unique_target($storeid, $scfg);
+    set_secret($storeid, $sensitive{'dhchap-key'});
+    return;
+}
+
+sub on_update_hook_full($class, $storeid, $scfg, $update, $delete, $sensitive) {
+    my %prospective = ($scfg->%*, $update->%*);
+    delete @prospective{$delete->@*} if $delete;
+    verify_nvme_nqn($prospective{subsysnqn});
+    _configured_portals(\%prospective);
+    _assert_unique_target($storeid, \%prospective);
+
+    my $old_key = file_read_firstline(secret_path($storeid));
+    my $key = exists($sensitive->{'dhchap-key'}) ? $sensitive->{'dhchap-key'} : $old_key;
+    _validate_secret($key);
+
+    # nvmet stores authentication on the global Host NQN object, which can be
+    # shared by multiple subsystems. Replacing an active key in place can make
+    # unrelated storages unrecoverable on their next reconnect. Until PVE can
+    # coordinate a rolling rotation on every node, fail before mutating the
+    # cluster-wide secret.
+    die "online NVMe DH-HMAC-CHAP key rotation is not supported; create a new storage/NQN\n"
+        if defined($old_key) && $key ne $old_key;
+
+    set_secret($storeid, $key) if exists($sensitive->{'dhchap-key'});
+    return;
+}
+
+sub on_delete_hook($class, $storeid, $scfg) {
+    eval { $class->deactivate_storage($storeid, $scfg) };
+    log_warn("failed to disconnect NVMe storage '$storeid': $@") if $@;
+    eval { PVE::Storage::LunCmd::NVMET::delete_target($scfg) };
+    log_warn("failed to remove NVMe target for '$storeid': $@") if $@;
+    delete_secret($storeid);
+    return;
+}
+
+my sub runtime_config_path($storeid) {
+    return "$runtime_dir/nvme-$storeid.json";
+}
+
+my sub write_runtime_config($storeid, $scfg, $hostnqn, $hostid, $key, $portals) {
+    my $ports = [
+        map {
+            {
+                transport => 'tcp',
+                traddr => $_->{address},
+                host_iface => $_->{host_iface},
+                trsvcid => "$_->{port}",
+                keep_alive_tmo => $scfg->{'nvme-keep-alive-tmo'} // 5,
+                reconnect_delay => $scfg->{'nvme-reconnect-delay'} // 2,
+                ctrl_loss_tmo => $scfg->{'nvme-ctrl-loss-tmo'} // 600,
+                ($scfg->{'nvme-nr-io-queues'}
+                    ? (nr_io_queues => $scfg->{'nvme-nr-io-queues'})
+                    : ()),
+            }
+        } $portals->@*
+    ];
+    my $config = [
+        {
+            hostnqn => $hostnqn,
+            hostid => $hostid,
+            dhchap_key => $key,
+            subsystems => [
+                {
+                    nqn => $scfg->{subsysnqn},
+                    ports => $ports,
+                    application => 'pve-storage',
+                },
+            ],
+        },
+    ];
+
+    make_path($runtime_dir, { mode => 0700 });
+    my $json = JSON->new->canonical->utf8->encode($config);
+    my $path = runtime_config_path($storeid);
+    file_set_contents($path, "$json\n", 0600);
+    return $path;
+}
+
+my sub controller_states($nqn) {
+    my $states = {};
+
+    opendir(my $dh, '/sys/class/nvme') or return $states;
+    while (defined(my $entry = readdir($dh))) {
+        next if $entry !~ $RE_NVME_CONTROLLER;
+        my $base = "/sys/class/nvme/$entry";
+        my $subsys = file_read_firstline("$base/subsysnqn");
+        next if !defined($subsys) || $subsys ne $nqn;
+        my $address = file_read_firstline("$base/address") // '';
+        my $state = file_read_firstline("$base/state") // 'unknown';
+        my $traddr = $address =~ $RE_TRADDR ? $+{value} : undef;
+        my $trsvcid = $address =~ $RE_TRSVCID ? $+{value} : undef;
+        my $host_iface = $address =~ $RE_HOST_IFACE_ADDRESS ? $+{value} : undef;
+        next if !defined($traddr) || !defined($trsvcid);
+        $states->{"$traddr:$trsvcid"} = {
+            device => "/dev/$entry",
+            host_iface => $host_iface,
+            state => $state,
+        };
+    }
+    closedir($dh);
+    return $states;
+}
+
+my sub portal_reachable($portal) {
+    my $socket = IO::Socket::IP->new(
+        PeerHost => $portal->{address},
+        PeerPort => $portal->{port},
+        Proto => 'tcp',
+        Timeout => 2,
+    );
+    return 0 if !$socket;
+    close($socket);
+    return 1;
+}
+
+sub _live_portal_count($states, $portals) {
+    my $live = 0;
+    for my $portal ($portals->@*) {
+        my $id = "$portal->{address}:$portal->{port}";
+        my $controller = $states->{$id};
+        $live++
+            if $controller
+            && $controller->{state} eq 'live'
+            && ($controller->{host_iface} // '') eq $portal->{host_iface};
+    }
+    return $live;
+}
+
+sub _connect_portal($config_path, $scfg, $portal) {
+    my $cmd = [
+        $nvme,
+        'connect',
+        '--config',
+        $config_path,
+        '--transport',
+        'tcp',
+        '--nqn',
+        $scfg->{subsysnqn},
+        '--traddr',
+        $portal->{address},
+        '--host-iface',
+        $portal->{host_iface},
+        '--trsvcid',
+        "$portal->{port}",
+        '--keep-alive-tmo',
+        $scfg->{'nvme-keep-alive-tmo'} // 5,
+        '--reconnect-delay',
+        $scfg->{'nvme-reconnect-delay'} // 2,
+        '--ctrl-loss-tmo',
+        $scfg->{'nvme-ctrl-loss-tmo'} // 600,
+    ];
+    if (my $queues = $scfg->{'nvme-nr-io-queues'}) {
+        push $cmd->@*, '--nr-io-queues', $queues;
+    }
+
+    run_command($cmd, timeout => 10, quiet => 1, errmsg => "NVMe/TCP connect failed");
+}
+
+my sub set_iopolicy($nqn, $policy) {
+    opendir(my $dh, '/sys/class/nvme-subsystem') or return;
+    while (defined(my $entry = readdir($dh))) {
+        next if $entry !~ $RE_NVME_SUBSYSTEM;
+        my $base = "/sys/class/nvme-subsystem/$entry";
+        my $subsys = file_read_firstline("$base/subsysnqn");
+        next if !defined($subsys) || $subsys ne $nqn;
+        my $fh = IO::File->new("$base/iopolicy", 'w')
+            or die "unable to set NVMe multipath policy: $!\n";
+        print $fh "$policy\n";
+        close($fh) or die "unable to set NVMe multipath policy: $!\n";
+    }
+    closedir($dh);
+}
+
+sub activate_storage($class, $storeid, $scfg, $cache = undef) {
+    $cache //= {};
+
+    die "nvme-cli is not installed\n" if !-x $nvme;
+    die "native NVMe multipath is disabled in the running kernel\n"
+        if (file_read_firstline('/sys/module/nvme_core/parameters/multipath') // 'N') ne 'Y';
+
+    _assert_unique_target($storeid, $scfg);
+    my $portals = _configured_portals($scfg);
+    my $states = controller_states($scfg->{subsysnqn});
+    my $force_reconcile =
+        delete($cache->{'zfsnvme-force-reconcile'}->{$storeid}) // 0;
+    my $live = _live_portal_count($states, $portals);
+
+    # This method is called by the periodic storage status loop. Once every
+    # configured controller is live, all lifecycle mutations are already
+    # reconciled by their individual operations, so avoid four SSH round-trips
+    # on every status poll. Missing namespaces explicitly force the slow path
+    # from activate_volume().
+    if (!$force_reconcile && $live == scalar($portals->@*)) {
+        die "missing NVMe DH-HMAC-CHAP key\n" if !-s secret_path($storeid);
+        set_iopolicy($scfg->{subsysnqn}, $scfg->{'nvme-iopolicy'} // 'round-robin');
+        return 1;
+    }
+
+    my $hostnqn = file_read_firstline('/etc/nvme/hostnqn')
+        // die "missing /etc/nvme/hostnqn\n";
+    my $hostid = file_read_firstline('/etc/nvme/hostid')
+        // die "missing /etc/nvme/hostid\n";
+    verify_nvme_nqn($hostnqn);
+    my $key = get_secret($storeid);
+    my $target_portals = [
+        map { "$_->{family},$_->{address},$_->{port}" } $portals->@*
+    ];
+
+    PVE::Storage::LunCmd::NVMET::ensure_target($scfg, $hostnqn, $target_portals);
+    PVE::Storage::LunCmd::NVMET::set_host_key($scfg, $hostnqn, $key);
+    PVE::Storage::LunCmd::NVMET::allow_host($scfg, $hostnqn);
+    PVE::Storage::LunCmd::NVMET::reconcile($scfg);
+
+    my $config_path =
+        write_runtime_config($storeid, $scfg, $hostnqn, $hostid, $key, $portals);
+    $states = controller_states($scfg->{subsysnqn});
+    for my $portal ($portals->@*) {
+        my $id = "$portal->{address}:$portal->{port}";
+        my $needs_rebind = 0;
+        if (my $controller = $states->{$id}) {
+            $needs_rebind = ($controller->{host_iface} // '') ne $portal->{host_iface};
+            next if !$needs_rebind && $controller->{state} ne 'dead';
+            run_command(
+                [$nvme, 'disconnect', '--device', $controller->{device}],
+                timeout => 5,
+                noerr => 1,
+                quiet => 1,
+            );
+        }
+        if (!portal_reachable($portal)) {
+            log_warn("NVMe/TCP portal '$id' is unreachable");
+            next;
+        }
+        eval { _connect_portal($config_path, $scfg, $portal) };
+        log_warn("$id: $@") if $@;
+
+        # Rebind controllers one at a time. Do not disconnect the next path
+        # until this one is live on its configured interface, so changing an
+        # interface mapping cannot cause an avoidable all-path outage.
+        if ($needs_rebind) {
+            my $rebound = 0;
+            for (my $attempt = 0; $attempt < 40; $attempt++) {
+                $states = controller_states($scfg->{subsysnqn});
+                my $controller = $states->{$id};
+                if (
+                    $controller
+                    && $controller->{state} eq 'live'
+                    && ($controller->{host_iface} // '') eq $portal->{host_iface}
+                ) {
+                    $rebound = 1;
+                    last;
+                }
+                select(undef, undef, undef, 0.25);
+            }
+            if (!$rebound) {
+                log_warn("NVMe/TCP portal '$id' did not rebind to '$portal->{host_iface}'");
+                last;
+            }
+        }
+    }
+
+    # Existing controllers can be in the middle of their kernel reconnect delay
+    # after a target restart. Do not create duplicates, but give that recovery
+    # cycle enough time to complete before declaring the storage unavailable.
+    for (my $attempt = 0; $attempt < 60; $attempt++) {
+        $states = controller_states($scfg->{subsysnqn});
+        last if _live_portal_count($states, $portals);
+        select(undef, undef, undef, 0.25);
+    }
+    $live = _live_portal_count($states, $portals);
+    die "no live NVMe/TCP path for storage '$storeid'\n" if !$live;
+    log_warn("storage '$storeid' is degraded: $live/" . scalar($portals->@*) . " paths live")
+        if $live < scalar($portals->@*);
+
+    set_iopolicy($scfg->{subsysnqn}, $scfg->{'nvme-iopolicy'} // 'round-robin');
+    return 1;
+}
+
+sub deactivate_storage($class, $storeid, $scfg, $cache = undef) {
+
+    run_command(
+        [$nvme, 'disconnect', '--nqn', $scfg->{subsysnqn}],
+        timeout => 15,
+        noerr => 1,
+        quiet => 1,
+    ) if -x $nvme;
+    unlink(runtime_config_path($storeid));
+    return 1;
+}
+
+sub path($class, $scfg, $volname, $storeid, $snapname = undef) {
+    die "direct access to snapshots not implemented\n" if defined($snapname);
+    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+    my $uuid = $class->zfs_get_lu_name($scfg, $name);
+    my $path = "/dev/disk/by-id/nvme-uuid.$uuid";
+    return ($path, $vmid, $vtype);
+}
+
+sub qemu_blockdev_options($class, $scfg, $storeid, $volname, $machine_version, $options) {
+    die "direct access to snapshots not implemented\n" if $options->{'snapshot-name'};
+    my ($path) = $class->path($scfg, $volname, $storeid);
+    return { driver => 'host_device', filename => $path };
+}
+
+sub activate_volume($class, $storeid, $scfg, $volname, $snapname, $cache = undef) {
+    die "unable to activate snapshot from remote zfs storage\n" if $snapname;
+    my ($path) = $class->path($scfg, $volname, $storeid);
+    if (!-b $path) {
+        $cache //= {};
+        $cache->{'zfsnvme-force-reconcile'}->{$storeid} = 1;
+        $class->activate_storage($storeid, $scfg, $cache);
+        for (my $attempt = 0; $attempt < 40 && !-b $path; $attempt++) {
+            select(undef, undef, undef, 0.25);
+        }
+    }
+    die "NVMe namespace for '$volname' did not appear\n" if !-b $path;
+    return 1;
+}
+
+sub deactivate_volume($class, $storeid, $scfg, $volname, $snapname, $cache = undef) {
+    die "unable to deactivate snapshot from remote zfs storage\n" if $snapname;
+    return 1;
+}
+
+sub volume_resize($class, $scfg, $storeid, $volname, $size, $running, $snapname) {
+    # QEMU's block_resize command explicitly cannot resize host block devices.
+    # Reject before changing the zvol/namespace, otherwise qemu-server would
+    # leave the backend larger while the running VM and its config keep the old
+    # size. A stopped VM is reopened with the new capacity on its next start.
+    die "online resize is not supported for NVMe/TCP block devices; stop the VM first\n"
+        if $running;
+    return $class->SUPER::volume_resize(
+        $scfg, $storeid, $volname, $size, $running, $snapname,
+    );
+}
+
+sub alloc_image($class, $storeid, $scfg, $vmid, $fmt, $name, $size) {
+    die "unsupported format '$fmt'" if $fmt ne 'raw';
+    die "illegal name '$name' - should be 'vm-$vmid-*'\n"
+        if $name && index($name, "vm-$vmid-") != 0;
+    my $volname = $name // $class->find_free_diskname($storeid, $scfg, $vmid, $fmt);
+
+    $class->zfs_create_zvol($scfg, $volname, $size);
+    eval {
+        my $uuid = $class->zfs_create_lu($scfg, $volname);
+        $class->zfs_add_lun_mapping_entry($scfg, $volname, $uuid);
+    };
+    if (my $err = $@) {
+        eval { $class->zfs_delete_zvol($scfg, $volname) };
+        warn "failed to clean up zvol '$volname': $@" if $@;
+        die $err;
+    }
+    return $volname;
+}
+
+1;
diff --git a/src/test/run_plugin_tests.pl b/src/test/run_plugin_tests.pl
index 8bce9d3..c341f4d 100755
--- a/src/test/run_plugin_tests.pl
+++ b/src/test/run_plugin_tests.pl
@@ -17,6 +17,7 @@ my $res = $harness->runtests(
     "get_subdir_test.pm",
     "filesystem_path_test.pm",
     "prune_backups_test.pm",
+    "zfsnvme_test.pm",
 );
 
 exit -1 if !$res || $res->{failed} || $res->{parse_errors};
diff --git a/src/test/zfsnvme_test.pm b/src/test/zfsnvme_test.pm
new file mode 100644
index 0000000..3e9e04b
--- /dev/null
+++ b/src/test/zfsnvme_test.pm
@@ -0,0 +1,350 @@
+package PVE::Storage::TestZFSNVMe;
+
+use v5.36;
+
+use lib qw(..);
+
+use PVE::Storage::LunCmd::LIO;
+use PVE::Storage::LunCmd::NVMET;
+use PVE::Storage::ZFSNVMePlugin;
+use PVE::Storage::ZFSPlugin;
+use Test::MockModule;
+use Test::More;
+
+is(
+    PVE::Storage::ZFSNVMePlugin::verify_nvme_nqn(
+        'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+    ),
+    'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+    'accepts a standards-based NQN',
+);
+
+ok(
+    !PVE::Storage::ZFSNVMePlugin::verify_nvme_nqn('not-an-nqn', 1),
+    'rejects an invalid NQN',
+);
+
+is_deeply(
+    PVE::Storage::ZFSNVMePlugin::parse_nvme_portals(
+        '10.90.1.11:4420,[fd00::11]:4421,10.90.2.11',
+    ),
+    [
+        { address => '10.90.1.11', port => 4420, family => 'ipv4' },
+        { address => 'fd00::11', port => 4421, family => 'ipv6' },
+        { address => '10.90.2.11', port => 4420, family => 'ipv4' },
+    ],
+    'parses and normalizes IPv4 and IPv6 portals',
+);
+
+eval {
+    PVE::Storage::ZFSNVMePlugin::parse_nvme_portals(
+        '10.90.1.11:4420,10.90.1.11:4420',
+    );
+};
+like($@, qr/duplicate NVMe\/TCP portal/, 'duplicate portal is rejected');
+
+ok(
+    !PVE::Storage::ZFSNVMePlugin::parse_nvme_portals('10.90.1.11:70000', 1),
+    'rejects an invalid TCP port',
+);
+
+ok(
+    !PVE::Storage::ZFSNVMePlugin::parse_nvme_portals(
+        join(',', map { "10.90.1.$_:4420" } 1 .. 17),
+        1,
+    ),
+    'limits the number of configured paths',
+);
+
+is_deeply(
+    PVE::Storage::ZFSNVMePlugin::_configured_portals({
+        'nvme-portals' => '10.90.1.11:4420,10.90.2.11:4421',
+        'nvme-host-ifaces' => 'ens20,ens21',
+    }),
+    [
+        {
+            address => '10.90.1.11',
+            port => 4420,
+            family => 'ipv4',
+            host_iface => 'ens20',
+        },
+        {
+            address => '10.90.2.11',
+            port => 4421,
+            family => 'ipv4',
+            host_iface => 'ens21',
+        },
+    ],
+    'binds each target portal to its configured local interface',
+);
+
+eval {
+    PVE::Storage::ZFSNVMePlugin::_configured_portals({
+        'nvme-portals' => '10.90.1.11,10.90.2.11',
+        'nvme-host-ifaces' => 'ens20',
+    });
+};
+like($@, qr/one interface for each/, 'portal and host-interface counts must match');
+
+eval {
+    PVE::Storage::ZFSNVMePlugin::_assert_unique_target(
+        'new-storage',
+        {
+            server => '192.0.2.10',
+            pool => 'tank/new',
+            subsysnqn => 'nqn.2026-07.example:duplicate',
+        },
+        {
+            ids => {
+                existing => {
+                    type => 'zfsnvme',
+                    server => '192.0.2.11',
+                    pool => 'tank/existing',
+                    subsysnqn => 'nqn.2026-07.example:duplicate',
+                },
+            },
+        },
+    );
+};
+like($@, qr/NQN is already used/, 'a subsystem NQN cannot be shared by storage definitions');
+
+eval {
+    PVE::Storage::ZFSNVMePlugin::_assert_unique_target(
+        'new-storage',
+        {
+            server => '192.0.2.10',
+            pool => 'tank/shared',
+            subsysnqn => 'nqn.2026-07.example:new',
+        },
+        {
+            ids => {
+                existing => {
+                    type => 'zfsnvme',
+                    server => '192.0.2.10',
+                    pool => 'tank/shared',
+                    subsysnqn => 'nqn.2026-07.example:existing',
+                },
+            },
+        },
+    );
+};
+like($@, qr/ZFS pool .* is already used/, 'a target pool cannot be shared by storage definitions');
+
+ok(
+    !PVE::Storage::ZFSNVMePlugin::parse_nvme_host_ifaces('ens20,not/an/interface', 1),
+    'rejects an invalid host interface name',
+);
+
+is(
+    PVE::Storage::ZFSNVMePlugin::_live_portal_count(
+        {
+            '10.90.1.11:4420' => { state => 'live', host_iface => 'eth0' },
+            '10.90.2.11:4420' => { state => 'live', host_iface => 'ens21' },
+        },
+        [
+            { address => '10.90.1.11', port => 4420, host_iface => 'ens20' },
+            { address => '10.90.2.11', port => 4420, host_iface => 'ens21' },
+        ],
+    ),
+    1,
+    'a live controller on the wrong interface is not an eligible path',
+);
+
+is(
+    PVE::Storage::ZFSNVMePlugin::_validate_secret(
+        'DHHC-1:01:YWJjZGVmZ2hpamtsbW5vcA==:',
+    ),
+    'DHHC-1:01:YWJjZGVmZ2hpamtsbW5vcA==:',
+    'accepts an NVMe DH-HMAC-CHAP secret representation',
+);
+
+eval { PVE::Storage::ZFSNVMePlugin::_validate_secret('plaintext') };
+like($@, qr/invalid NVMe DH-HMAC-CHAP/, 'rejects a plaintext secret');
+
+my $nvme_mock = Test::MockModule->new('PVE::Storage::ZFSNVMePlugin');
+$nvme_mock->redefine(zfs_get_lu_name => sub { return '12345678-1234-1234-1234-123456789abc' });
+
+my $scfg = {};
+my ($path, $vmid, $vtype) = PVE::Storage::ZFSNVMePlugin->path(
+    $scfg,
+    'vm-100-disk-0',
+    'nvmetest',
+);
+is(
+    $path,
+    '/dev/disk/by-id/nvme-uuid.12345678-1234-1234-1234-123456789abc',
+    'uses the stable namespace UUID symlink',
+);
+is($vmid, 100, 'returns the VM owner');
+is($vtype, 'images', 'returns the volume type');
+is_deeply(
+    PVE::Storage::ZFSNVMePlugin->qemu_blockdev_options(
+        $scfg,
+        'nvmetest',
+        'vm-100-disk-0',
+        undef,
+        {},
+    ),
+    {
+        driver => 'host_device',
+        filename => '/dev/disk/by-id/nvme-uuid.12345678-1234-1234-1234-123456789abc',
+    },
+    'uses the QEMU host_device driver',
+);
+
+my $lio_mock = Test::MockModule->new('PVE::Storage::LunCmd::LIO');
+my @provider_args;
+$lio_mock->redefine(
+    run_lun_command => sub {
+        @provider_args = @_;
+        return 'ok';
+    },
+);
+is(
+    PVE::Storage::ZFSPlugin->zfs_request(
+        { iscsiprovider => 'LIO' },
+        10,
+        'add_view',
+        'guid',
+    ),
+    'ok',
+    'legacy LIO dispatch still works',
+);
+is(ref($provider_args[0]), 'HASH', 'provider dispatch does not inject a class argument');
+is($provider_args[1], 10, 'provider timeout argument is preserved');
+is($provider_args[2], 'add_view', 'provider method argument is preserved');
+is($provider_args[3], 'guid', 'provider parameters are preserved');
+
+my @connect_cmd;
+$nvme_mock->redefine(
+    run_command => sub {
+        @connect_cmd = $_[0]->@*;
+        return 0;
+    },
+);
+PVE::Storage::ZFSNVMePlugin::_connect_portal(
+    '/run/pve-storage/test.json',
+    { subsysnqn => 'nqn.2026-07.example:test' },
+    { address => '10.90.2.11', port => 4420, host_iface => 'ens21' },
+);
+is_deeply(
+    [@connect_cmd[10, 11]],
+    ['--host-iface', 'ens21'],
+    'nvme connect is explicitly bound to the configured data interface',
+);
+
+$nvme_mock->redefine(
+    zfs_request => sub {
+        my ($class, $config, $timeout, $method, @params) = @_;
+        return join(
+            "\n",
+            "tank/vm-100-disk-0\t1048576\t-\tvolume\t-",
+            "tank/vm-200-disk-0\t1048576\t-\tvolume\t-",
+            "tank/vm-300-disk-0\t1048576\t-\tvolume\t-",
+            '',
+        ) if $method eq 'list';
+        return join(
+            "\n",
+            "tank/vm-100-disk-0\tnqn.2026-07.example:owned\tlocal",
+            "tank/vm-200-disk-0\t-\t-",
+            "tank/vm-300-disk-0\tnqn.2026-07.example:owned\tinherited from tank",
+            '',
+        ) if $method eq 'get';
+        die "unexpected mocked ZFS method '$method'\n";
+    },
+);
+my $owned_zvols = PVE::Storage::ZFSNVMePlugin->zfs_list_zvol(
+    { pool => 'tank', subsysnqn => 'nqn.2026-07.example:owned' },
+);
+is_deeply(
+    [sort keys $owned_zvols->%*],
+    ['vm-100-disk-0'],
+    'volume listing requires a local or received ownership property',
+);
+
+eval {
+    PVE::Storage::ZFSNVMePlugin->volume_resize(
+        {}, 'nvmetest', 'vm-100-disk-0', 2 * 1024 * 1024 * 1024, 1, undef,
+    );
+};
+like(
+    $@,
+    qr/online resize is not supported.*stop the VM first/,
+    'online resize is rejected before mutating the backend',
+);
+
+my $update_key = 'DHHC-1:01:dXBkYXRlLXRlc3Qta2V5:';
+$nvme_mock->redefine(file_read_firstline => sub { return $update_key });
+my $zfs_parent_mock = Test::MockModule->new('PVE::Storage::ZFSPlugin');
+$zfs_parent_mock->redefine(check_config => sub { return $_[2] });
+eval {
+    PVE::Storage::ZFSNVMePlugin->check_config(
+        'nvmetest',
+        { 'nvme-host-ifaces' => 'ens20,ens21' },
+        0,
+        1,
+    );
+};
+is($@, '', 'check_config accepts a valid partial update');
+
+{
+    no warnings 'redefine';
+    local *PVE::Storage::config = sub { return { ids => {} } };
+    eval {
+        PVE::Storage::ZFSNVMePlugin->on_update_hook_full(
+            'nvmetest',
+            {
+                subsysnqn => 'nqn.2026-07.example:test',
+                'nvme-portals' => '10.90.1.11,10.90.2.11',
+            },
+            { 'nvme-host-ifaces' => 'ens20,ens21' },
+            undef,
+            {},
+        );
+    };
+    is($@, '', 'partial update with no delete list validates against the current config');
+}
+
+my $nvmet_mock = Test::MockModule->new('PVE::Storage::LunCmd::NVMET');
+my @helper_calls;
+$nvmet_mock->redefine(
+    run_command => sub {
+        my ($cmd, %opts) = @_;
+        push @helper_calls, { cmd => $cmd, %opts };
+        return 0;
+    },
+);
+PVE::Storage::LunCmd::NVMET::ensure_target(
+    {
+        server => '192.0.2.10',
+        subsysnqn => 'nqn.2026-07.example:test',
+        pool => 'tank/pve-nvme',
+    },
+    'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+    ['ipv4,10.90.1.11,4420'],
+);
+like(
+    $helper_calls[0]->{input},
+    qr/current_model.*current_serial.*refusing to take over existing NVMe subsystem/s,
+    'target reconcile verifies model and deterministic serial before taking over a subsystem',
+);
+
+my ($key_cmd, %key_opts);
+$nvmet_mock->redefine(
+    run_command => sub {
+        ($key_cmd, %key_opts) = @_;
+        return 0;
+    },
+);
+my $test_key = 'DHHC-1:01:bm90LWEtcmVhbC1rZXk=:';
+PVE::Storage::LunCmd::NVMET::set_host_key(
+    { server => '192.0.2.10' },
+    'nqn.2014-08.org.nvmexpress:uuid:12345678-1234-1234-1234-123456789abc',
+    $test_key,
+);
+unlike(join(' ', $key_cmd->@*), qr/\Q$test_key\E/, 'DHCHAP key is absent from the process argv');
+is($key_opts{input}, "$test_key\n", 'DHCHAP key is provided through standard input');
+
+done_testing();
+
+1;
-- 
2.54.0.windows.1