From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 1EB291FF13C for ; Thu, 30 Apr 2026 19:34:26 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0E93416678; Thu, 30 Apr 2026 19:33:39 +0200 (CEST) From: Mira Limbeck To: pve-devel@lists.proxmox.com Subject: [POC v2 storage 14/15] mapping: add zfspool plugin Date: Thu, 30 Apr 2026 19:27:12 +0200 Message-ID: <20260430173220.441001-15-m.limbeck@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260430173220.441001-1-m.limbeck@proxmox.com> References: <20260430173220.441001-1-m.limbeck@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.544 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods RDNS_NONE 0.793 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Message-ID-Hash: PO55YTK6RNE6JTCYPSGE3IW6S4T6STYB X-Message-ID-Hash: PO55YTK6RNE6JTCYPSGE3IW6S4T6STYB X-MailFrom: mira@nena.proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: proof of concept to see if the chosen mapping abstraction works for other storages as well. zfspool mapping plugin enables replication between nodes with different zpools. Signed-off-by: Mira Limbeck --- This is sent as a POC since the motivation behind it was mainly to see if the mapping plugin abstraction works for other storages as well. Please note that testing wasn't in-depth enough to make sure all the uses are handled and work reliably with mappings. But in my tests (mostly replication) it worked nicely. src/PVE/Storage/Mapping.pm | 2 + src/PVE/Storage/Mapping/Makefile | 3 +- src/PVE/Storage/Mapping/Plugin.pm | 2 + src/PVE/Storage/Mapping/ZFSPool.pm | 48 +++++++++++ src/PVE/Storage/ZFSPoolPlugin.pm | 133 ++++++++++++++++++++--------- 5 files changed, 148 insertions(+), 40 deletions(-) create mode 100644 src/PVE/Storage/Mapping/ZFSPool.pm diff --git a/src/PVE/Storage/Mapping.pm b/src/PVE/Storage/Mapping.pm index b607156..917ce32 100644 --- a/src/PVE/Storage/Mapping.pm +++ b/src/PVE/Storage/Mapping.pm @@ -3,9 +3,11 @@ package PVE::Storage::Mapping; use PVE::JSONSchema; use PVE::Storage::Mapping::ISCSI; +use PVE::Storage::Mapping::ZFSPool; use PVE::Storage::Mapping::Plugin; PVE::Storage::Mapping::ISCSI->register(); +PVE::Storage::Mapping::ZFSPool->register(); PVE::Storage::Mapping::Plugin->init(property_isolation => 1); sub find_mapping_on_current_node { diff --git a/src/PVE/Storage/Mapping/Makefile b/src/PVE/Storage/Mapping/Makefile index 25fae16..0d43f52 100644 --- a/src/PVE/Storage/Mapping/Makefile +++ b/src/PVE/Storage/Mapping/Makefile @@ -1,6 +1,7 @@ SOURCES= \ Plugin.pm \ - ISCSI.pm + ISCSI.pm \ + ZFSPool.pm .PHONY: install install: diff --git a/src/PVE/Storage/Mapping/Plugin.pm b/src/PVE/Storage/Mapping/Plugin.pm index a0d3198..fe5ecd0 100644 --- a/src/PVE/Storage/Mapping/Plugin.pm +++ b/src/PVE/Storage/Mapping/Plugin.pm @@ -4,6 +4,8 @@ use strict; use warnings; use PVE::Storage::Mapping::ISCSI; +use PVE::Storage::Mapping::ZFSPool; + use PVE::INotify; use PVE::JSONSchema; use PVE::Cluster qw( diff --git a/src/PVE/Storage/Mapping/ZFSPool.pm b/src/PVE/Storage/Mapping/ZFSPool.pm new file mode 100644 index 0000000..b335bb6 --- /dev/null +++ b/src/PVE/Storage/Mapping/ZFSPool.pm @@ -0,0 +1,48 @@ +package PVE::Storage::Mapping::ZFSPool; + +use strict; +use warnings; + +use PVE::JSONSchema qw(get_standard_option); +use PVE::Storage::Mapping::Plugin; +use base qw(PVE::Storage::Mapping::Plugin); + +sub type { + return 'zfspool'; +} + +my $map_fmt = { + node => get_standard_option('pve-node'), + pool => { + type => 'string', + description => 'Local ZFS pool.', + }, +}; + +sub properties { + return { + map => { + type => 'array', + description => 'A list of maps.', + optional => 1, + items => { + type => 'string', + format => $map_fmt, + }, + }, + }; +} + +sub options { + return { + description => { optional => 1 }, + map => { optional => 1 }, + }; +} + +sub get_map_format { + return { $map_fmt->%* }; +} + +1; + diff --git a/src/PVE/Storage/ZFSPoolPlugin.pm b/src/PVE/Storage/ZFSPoolPlugin.pm index 3b3456b..4a06876 100644 --- a/src/PVE/Storage/ZFSPoolPlugin.pm +++ b/src/PVE/Storage/ZFSPoolPlugin.pm @@ -46,7 +46,7 @@ sub properties { sub options { return { - pool => { fixed => 1 }, + pool => { fixed => 1, optional => 1 }, blocksize => { optional => 1 }, sparse => { optional => 1 }, nodes => { optional => 1 }, @@ -54,9 +54,38 @@ sub options { content => { optional => 1 }, bwlimit => { optional => 1 }, mountpoint => { optional => 1 }, + mapping => { optional => 1 }, }; } +my $get_local_pool = sub { + my ($scfg) = @_; + + die "neither 'pool' nor 'mapping' defined\n" + if !defined($scfg->{pool}) && !defined($scfg->{mapping}); + + my $pool = undef; + + # prefer mapping over direct pool config + if ($scfg->{mapping}) { + my $local_mappings = + PVE::Storage::Mapping::find_mapping_on_current_node($scfg->{mapping}); + die "no ZFSPool per-node entries found for mapping '$scfg->{mapping}'\n" + if !defined($local_mappings) || !$local_mappings->@*; + + for my $mapping ($local_mappings->@*) { + die "different zfs pools configured for the same node\n" + if defined($pool) && $pool ne $mapping->{pool}; + + $pool = $mapping->{pool}; + } + } else { + $pool = $scfg->{pool}; + } + + return $pool; +}; + # static zfs helper methods sub zfs_parse_zvol_list { @@ -124,7 +153,7 @@ sub on_add_hook { # ignore failure, pool might currently not be imported my $mountpoint; eval { - my $res = $class->zfs_get_properties($scfg, 'mountpoint', $scfg->{pool}, 1); + my $res = $class->zfs_get_properties($scfg, 'mountpoint', $get_local_pool->($scfg), 1); $mountpoint = PVE::Storage::Plugin::verify_path($res, 1) if defined($res); }; @@ -145,14 +174,15 @@ sub path { my ($vtype, $name, $vmid) = $class->parse_volname($volname); + my $pool = $get_local_pool->($scfg); my $path = ''; - my $mountpoint = $scfg->{mountpoint} // "/$scfg->{pool}"; + my $mountpoint = $scfg->{mountpoint} // "/$pool"; if ($vtype eq "images") { if ($name =~ m/^subvol-/ || $name =~ m/^basevol-/) { $path = "$mountpoint/$name"; } else { - $path = "/dev/zvol/$scfg->{pool}/$name"; + $path = "/dev/zvol/$pool/$name"; } $path .= "\@$snapname" if defined($snapname); } else { @@ -315,7 +345,7 @@ sub zfs_get_pool_stats { my $available = 0; my $used = 0; - my @lines = $class->zfs_get_properties($scfg, 'available,used', $scfg->{pool}); + my @lines = $class->zfs_get_properties($scfg, 'available,used', $get_local_pool->($scfg)); if ($lines[0] =~ /^(\d+)$/) { $available = $1; @@ -331,6 +361,8 @@ sub zfs_get_pool_stats { sub zfs_create_zvol { my ($class, $scfg, $zvol, $size) = @_; + my $pool = $get_local_pool->($scfg); + # always align size to 1M as workaround until # https://github.com/zfsonlinux/zfs/issues/8541 is solved my $padding = (1024 - $size % 1024) % 1024; @@ -342,7 +374,7 @@ sub zfs_create_zvol { push @$cmd, '-b', $scfg->{blocksize} if $scfg->{blocksize}; - push @$cmd, '-V', "${size}k", "$scfg->{pool}/$zvol"; + push @$cmd, '-V', "${size}k", "$pool/$zvol"; $class->zfs_request($scfg, undef, @$cmd); } @@ -350,7 +382,8 @@ sub zfs_create_zvol { sub zfs_create_subvol { my ($class, $scfg, $volname, $size) = @_; - my $dataset = "$scfg->{pool}/$volname"; + my $pool = $get_local_pool->($scfg); + my $dataset = "$pool/$volname"; my $quota = $size ? "${size}k" : "none"; my $cmd = @@ -362,11 +395,12 @@ sub zfs_create_subvol { sub zfs_delete_zvol { my ($class, $scfg, $zvol) = @_; + my $pool = $get_local_pool->($scfg); my $err; for (my $i = 0; $i < 6; $i++) { - eval { $class->zfs_request($scfg, undef, 'destroy', '-r', "$scfg->{pool}/$zvol"); }; + eval { $class->zfs_request($scfg, undef, 'destroy', '-r', "$pool/$zvol"); }; if ($err = $@) { if ($err =~ m/dataset is busy/) { sleep(1); @@ -387,6 +421,8 @@ sub zfs_delete_zvol { sub zfs_list_zvol { my ($class, $scfg) = @_; + my $pool = $get_local_pool->($scfg); + my $text = $class->zfs_request( $scfg, 10, @@ -397,18 +433,18 @@ sub zfs_list_zvol { 'volume,filesystem', '-d1', '-Hp', - $scfg->{pool}, + $pool, ); # It's still required to have zfs_parse_zvol_list filter by pool, because -d1 lists - # $scfg->{pool} too and while unlikely, it could be named to be mistaken for a volume. - my $zvols = zfs_parse_zvol_list($text, $scfg->{pool}); + # $pool too and while unlikely, it could be named to be mistaken for a volume. + my $zvols = zfs_parse_zvol_list($text, $pool); return {} if !$zvols; my $list = {}; foreach my $zvol (@$zvols) { my $name = $zvol->{name}; my $parent = $zvol->{origin}; - if ($zvol->{origin} && $zvol->{origin} =~ m/^$scfg->{pool}\/(\S+)$/) { + if ($zvol->{origin} && $zvol->{origin} =~ m/^$pool\/(\S+)$/) { $parent = $1; } @@ -430,7 +466,8 @@ sub zfs_get_sorted_snapshot_list { my @params = ('-H', '-r', '-t', 'snapshot', '-o', 'name', $sort_params->@*); my $vname = ($class->parse_volname($volname))[1]; - push @params, "$scfg->{pool}\/$vname"; + my $pool = $get_local_pool->($scfg); + push @params, "$pool\/$vname"; my $text = $class->zfs_request($scfg, undef, 'list', @params); my @snapshots = split(/\n/, $text); @@ -466,9 +503,9 @@ sub volume_size_info { my (undef, $vname, undef, $parent, undef, undef, $format) = $class->parse_volname($volname); + my $pool = $get_local_pool->($scfg); my $attr = $format eq 'subvol' ? 'refquota' : 'volsize'; - my ($size, $used) = - $class->zfs_get_properties($scfg, "$attr,usedbydataset", "$scfg->{pool}/$vname"); + my ($size, $used) = $class->zfs_get_properties($scfg, "$attr,usedbydataset", "$pool/$vname"); $used = ($used =~ /^(\d+)$/) ? $1 : 0; @@ -483,7 +520,8 @@ sub volume_snapshot { my ($class, $scfg, $storeid, $volname, $snap) = @_; my (undef, $vname, undef, undef, undef, undef, $format) = $class->parse_volname($volname); - my $snapshot_name = "$scfg->{pool}/$vname\@$snap"; + my $pool = $get_local_pool->($scfg); + my $snapshot_name = "$pool/$vname\@$snap"; $class->zfs_request($scfg, undef, 'snapshot', $snapshot_name); @@ -491,7 +529,7 @@ sub volume_snapshot { # does not track this property for snapshosts and consequently does not roll # it back. so track this information manually. if ($format eq 'subvol') { - my $refquota = $class->zfs_get_properties($scfg, 'refquota', "$scfg->{pool}/$vname"); + my $refquota = $class->zfs_get_properties($scfg, 'refquota', "$pool/$vname"); $class->zfs_request( $scfg, @@ -507,16 +545,18 @@ sub volume_snapshot_delete { my ($class, $scfg, $storeid, $volname, $snap, $running) = @_; my $vname = ($class->parse_volname($volname))[1]; + my $pool = $get_local_pool->($scfg); $class->deactivate_volume($storeid, $scfg, $vname, $snap, {}); - $class->zfs_request($scfg, undef, 'destroy', "$scfg->{pool}/$vname\@$snap"); + $class->zfs_request($scfg, undef, 'destroy', "$pool/$vname\@$snap"); } sub volume_snapshot_rollback { my ($class, $scfg, $storeid, $volname, $snap) = @_; my (undef, $vname, undef, undef, undef, undef, $format) = $class->parse_volname($volname); - my $snapshot_name = "$scfg->{pool}/$vname\@$snap"; + my $pool = $get_local_pool->($scfg); + my $snapshot_name = "$pool/$vname\@$snap"; my $msg = $class->zfs_request($scfg, undef, 'rollback', $snapshot_name); @@ -527,7 +567,7 @@ sub volume_snapshot_rollback { if ($refquota =~ m/^\d+$/) { $class->zfs_request( - $scfg, undef, 'set', "refquota=${refquota}", "$scfg->{pool}/$vname", + $scfg, undef, 'set', "refquota=${refquota}", "$pool/$vname", ); } elsif ($refquota ne "-") { # refquota user property was set, but not a number -> warn @@ -539,7 +579,7 @@ sub volume_snapshot_rollback { # caches, they get mounted in activate volume again # see zfs bug #10931 https://github.com/openzfs/zfs/issues/10931 if ($format eq 'subvol') { - eval { $class->zfs_request($scfg, undef, 'unmount', "$scfg->{pool}/$vname"); }; + eval { $class->zfs_request($scfg, undef, 'unmount', "$pool/$vname"); }; if (my $err = $@) { die $err if $err !~ m/not currently mounted$/; } @@ -582,7 +622,8 @@ sub volume_snapshot_info { my @params = ('-Hp', '-r', '-t', 'snapshot', '-o', 'name,guid,creation'); my $vname = ($class->parse_volname($volname))[1]; - push @params, "$scfg->{pool}\/$vname"; + my $pool = $get_local_pool->($scfg); + push @params, "$pool\/$vname"; my $text = $class->zfs_request($scfg, undef, 'list', @params); my @lines = split(/\n/, $text); @@ -619,7 +660,7 @@ sub activate_storage { my ($class, $storeid, $scfg, $cache) = @_; # Note: $scfg->{pool} can include dataset / - my $dataset = $scfg->{pool}; + my $dataset = $get_local_pool->($scfg); my $pool = ($dataset =~ s!/.*$!!r); return 1 if dataset_mounted_heuristic($dataset); # early return @@ -657,13 +698,14 @@ sub activate_volume { return 1 if defined($snapname); my (undef, $dataset, undef, undef, undef, undef, $format) = $class->parse_volname($volname); + my $pool = $get_local_pool->($scfg); if ($format eq 'raw') { $class->zfs_wait_for_zvol_link($scfg, $volname); } elsif ($format eq 'subvol') { - my $mounted = $class->zfs_get_properties($scfg, 'mounted', "$scfg->{pool}/$dataset"); + my $mounted = $class->zfs_get_properties($scfg, 'mounted', "$pool/$dataset"); if ($mounted !~ m/^yes$/) { - $class->zfs_request($scfg, undef, 'mount', "$scfg->{pool}/$dataset"); + $class->zfs_request($scfg, undef, 'mount', "$pool/$dataset"); } } @@ -686,28 +728,25 @@ sub clone_image { die "clone_image only works on base images\n" if !$isBase; my $name = $class->find_free_diskname($storeid, $scfg, $vmid, $format); + my $pool = $get_local_pool->($scfg); if ($format eq 'subvol') { my $size = $class->zfs_request( - $scfg, undef, 'list', '-Hp', '-o', 'refquota', "$scfg->{pool}/$basename", + $scfg, undef, 'list', '-Hp', '-o', 'refquota', "$pool/$basename", ); chomp($size); $class->zfs_request( $scfg, undef, 'clone', - "$scfg->{pool}/$basename\@$snap", - "$scfg->{pool}/$name", + "$pool/$basename\@$snap", + "$pool/$name", '-o', "refquota=$size", ); } else { $class->zfs_request( - $scfg, - undef, - 'clone', - "$scfg->{pool}/$basename\@$snap", - "$scfg->{pool}/$name", + $scfg, undef, 'clone', "$pool/$basename\@$snap", "$pool/$name", ); } @@ -731,8 +770,9 @@ sub create_base { $newname =~ s/^vm-/base-/; } my $newvolname = $basename ? "$basename/$newname" : "$newname"; + my $pool = $get_local_pool->($scfg); - $class->zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname"); + $class->zfs_request($scfg, undef, 'rename', "$pool/$name", "$pool/$newname"); my $running = undef; #fixme : is create_base always offline ? @@ -756,7 +796,8 @@ sub volume_resize { $new_size = $new_size + $padding; } - $class->zfs_request($scfg, undef, 'set', "$attr=${new_size}k", "$scfg->{pool}/$vname"); + my $pool = $get_local_pool->($scfg); + $class->zfs_request($scfg, undef, 'set', "$attr=${new_size}k", "$pool/$vname"); return $new_size; } @@ -822,7 +863,8 @@ sub volume_export { my $arg = $with_snapshots ? '-I' : '-i'; push @$cmd, $arg, $base_snapshot; } - push @$cmd, '--', "$scfg->{pool}/$dataset\@$snapshot"; + my $pool = $get_local_pool->($scfg); + push @$cmd, '--', "$pool/$dataset\@$snapshot"; run_command($cmd, output => $fd); @@ -862,8 +904,9 @@ sub volume_import { my (undef, $dataset, $vmid, undef, undef, undef, $volume_format) = $class->parse_volname($volname); + my $pool = $get_local_pool->($scfg); - my $zfspath = "$scfg->{pool}/$dataset"; + my $zfspath = "$pool/$dataset"; my $suffix = defined($base_snapshot) ? "\@$base_snapshot" : ''; my $exists = 0 == run_command( ['zfs', 'get', '-H', 'name', $zfspath . $suffix], @@ -876,7 +919,7 @@ sub volume_import { die "volume '$zfspath' already exists\n" if !$allow_rename; warn "volume '$zfspath' already exists - importing with a different name\n"; $dataset = $class->find_free_diskname($storeid, $scfg, $vmid, $volume_format); - $zfspath = "$scfg->{pool}/$dataset"; + $zfspath = "$pool/$dataset"; } eval { run_command(['zfs', 'recv', '-F', '--', $zfspath], input => "<&$fd") }; @@ -909,7 +952,7 @@ sub rename_volume { $target_volname = $class->find_free_diskname($storeid, $scfg, $target_vmid, $format) if !$target_volname; - my $pool = $scfg->{pool}; + my $pool = $get_local_pool->($scfg); my $source_zfspath = "${pool}/${source_image}"; my $target_zfspath = "${pool}/${target_volname}"; @@ -933,4 +976,16 @@ sub rename_snapshot { die "rename_snapshot is not supported for $class"; } +sub check_config { + my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_; + + my $checked = $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck); + + # check if either target or mapping is set + die "zfspool storage '$sectionId' has neither 'pool' nor 'mapping' defined\n" + if !defined($checked->{pool}) && !defined($checked->{mapping}); + + return $checked; +} + 1; -- 2.47.3