[POC v2 storage 14/15] mapping: add zfspool plugin

[Mira Limbeck <m.limbeck@proxmox.com>]

proof of concept to see if the chosen mapping abstraction works for
other storages as well.
zfspool mapping plugin enables replication between nodes with different
zpools.

Signed-off-by: Mira Limbeck <m.limbeck@proxmox.com>
---
This is sent as a POC since the motivation behind it was mainly to see
if the mapping plugin abstraction works for other storages as well.
Please note that testing wasn't in-depth enough to make sure
all the uses are handled and work reliably with mappings.

But in my tests (mostly replication) it worked nicely.


 src/PVE/Storage/Mapping.pm         |   2 +
 src/PVE/Storage/Mapping/Makefile   |   3 +-
 src/PVE/Storage/Mapping/Plugin.pm  |   2 +
 src/PVE/Storage/Mapping/ZFSPool.pm |  48 +++++++++++
 src/PVE/Storage/ZFSPoolPlugin.pm   | 133 ++++++++++++++++++++---------
 5 files changed, 148 insertions(+), 40 deletions(-)
 create mode 100644 src/PVE/Storage/Mapping/ZFSPool.pm

diff --git a/src/PVE/Storage/Mapping.pm b/src/PVE/Storage/Mapping.pm
index b607156..917ce32 100644
--- a/src/PVE/Storage/Mapping.pm
+++ b/src/PVE/Storage/Mapping.pm
@@ -3,9 +3,11 @@ package PVE::Storage::Mapping;
 use PVE::JSONSchema;
 
 use PVE::Storage::Mapping::ISCSI;
+use PVE::Storage::Mapping::ZFSPool;
 use PVE::Storage::Mapping::Plugin;
 
 PVE::Storage::Mapping::ISCSI->register();
+PVE::Storage::Mapping::ZFSPool->register();
 PVE::Storage::Mapping::Plugin->init(property_isolation => 1);
 
 sub find_mapping_on_current_node {
diff --git a/src/PVE/Storage/Mapping/Makefile b/src/PVE/Storage/Mapping/Makefile
index 25fae16..0d43f52 100644
--- a/src/PVE/Storage/Mapping/Makefile
+++ b/src/PVE/Storage/Mapping/Makefile
@@ -1,6 +1,7 @@
 SOURCES= \
 	Plugin.pm \
-	ISCSI.pm
+	ISCSI.pm \
+	ZFSPool.pm
 
 .PHONY: install
 install:
diff --git a/src/PVE/Storage/Mapping/Plugin.pm b/src/PVE/Storage/Mapping/Plugin.pm
index a0d3198..fe5ecd0 100644
--- a/src/PVE/Storage/Mapping/Plugin.pm
+++ b/src/PVE/Storage/Mapping/Plugin.pm
@@ -4,6 +4,8 @@ use strict;
 use warnings;
 
 use PVE::Storage::Mapping::ISCSI;
+use PVE::Storage::Mapping::ZFSPool;
+
 use PVE::INotify;
 use PVE::JSONSchema;
 use PVE::Cluster qw(
diff --git a/src/PVE/Storage/Mapping/ZFSPool.pm b/src/PVE/Storage/Mapping/ZFSPool.pm
new file mode 100644
index 0000000..b335bb6
--- /dev/null
+++ b/src/PVE/Storage/Mapping/ZFSPool.pm
@@ -0,0 +1,48 @@
+package PVE::Storage::Mapping::ZFSPool;
+
+use strict;
+use warnings;
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::Storage::Mapping::Plugin;
+use base qw(PVE::Storage::Mapping::Plugin);
+
+sub type {
+    return 'zfspool';
+}
+
+my $map_fmt = {
+    node => get_standard_option('pve-node'),
+    pool => {
+        type => 'string',
+        description => 'Local ZFS pool.',
+    },
+};
+
+sub properties {
+    return {
+        map => {
+            type => 'array',
+            description => 'A list of maps.',
+            optional => 1,
+            items => {
+                type => 'string',
+                format => $map_fmt,
+            },
+        },
+    };
+}
+
+sub options {
+    return {
+        description => { optional => 1 },
+        map => { optional => 1 },
+    };
+}
+
+sub get_map_format {
+    return { $map_fmt->%* };
+}
+
+1;
+
diff --git a/src/PVE/Storage/ZFSPoolPlugin.pm b/src/PVE/Storage/ZFSPoolPlugin.pm
index 3b3456b..4a06876 100644
--- a/src/PVE/Storage/ZFSPoolPlugin.pm
+++ b/src/PVE/Storage/ZFSPoolPlugin.pm
@@ -46,7 +46,7 @@ sub properties {
 
 sub options {
     return {
-        pool => { fixed => 1 },
+        pool => { fixed => 1, optional => 1 },
         blocksize => { optional => 1 },
         sparse => { optional => 1 },
         nodes => { optional => 1 },
@@ -54,9 +54,38 @@ sub options {
         content => { optional => 1 },
         bwlimit => { optional => 1 },
         mountpoint => { optional => 1 },
+        mapping => { optional => 1 },
     };
 }
 
+my $get_local_pool = sub {
+    my ($scfg) = @_;
+
+    die "neither 'pool' nor 'mapping' defined\n"
+        if !defined($scfg->{pool}) && !defined($scfg->{mapping});
+
+    my $pool = undef;
+
+    # prefer mapping over direct pool config
+    if ($scfg->{mapping}) {
+        my $local_mappings =
+            PVE::Storage::Mapping::find_mapping_on_current_node($scfg->{mapping});
+        die "no ZFSPool per-node entries found for mapping '$scfg->{mapping}'\n"
+            if !defined($local_mappings) || !$local_mappings->@*;
+
+        for my $mapping ($local_mappings->@*) {
+            die "different zfs pools configured for the same node\n"
+                if defined($pool) && $pool ne $mapping->{pool};
+
+            $pool = $mapping->{pool};
+        }
+    } else {
+        $pool = $scfg->{pool};
+    }
+
+    return $pool;
+};
+
 # static zfs helper methods
 
 sub zfs_parse_zvol_list {
@@ -124,7 +153,7 @@ sub on_add_hook {
     # ignore failure, pool might currently not be imported
     my $mountpoint;
     eval {
-        my $res = $class->zfs_get_properties($scfg, 'mountpoint', $scfg->{pool}, 1);
+        my $res = $class->zfs_get_properties($scfg, 'mountpoint', $get_local_pool->($scfg), 1);
         $mountpoint = PVE::Storage::Plugin::verify_path($res, 1) if defined($res);
     };
 
@@ -145,14 +174,15 @@ sub path {
 
     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
 
+    my $pool = $get_local_pool->($scfg);
     my $path = '';
-    my $mountpoint = $scfg->{mountpoint} // "/$scfg->{pool}";
+    my $mountpoint = $scfg->{mountpoint} // "/$pool";
 
     if ($vtype eq "images") {
         if ($name =~ m/^subvol-/ || $name =~ m/^basevol-/) {
             $path = "$mountpoint/$name";
         } else {
-            $path = "/dev/zvol/$scfg->{pool}/$name";
+            $path = "/dev/zvol/$pool/$name";
         }
         $path .= "\@$snapname" if defined($snapname);
     } else {
@@ -315,7 +345,7 @@ sub zfs_get_pool_stats {
     my $available = 0;
     my $used = 0;
 
-    my @lines = $class->zfs_get_properties($scfg, 'available,used', $scfg->{pool});
+    my @lines = $class->zfs_get_properties($scfg, 'available,used', $get_local_pool->($scfg));
 
     if ($lines[0] =~ /^(\d+)$/) {
         $available = $1;
@@ -331,6 +361,8 @@ sub zfs_get_pool_stats {
 sub zfs_create_zvol {
     my ($class, $scfg, $zvol, $size) = @_;
 
+    my $pool = $get_local_pool->($scfg);
+
     # always align size to 1M as workaround until
     # https://github.com/zfsonlinux/zfs/issues/8541 is solved
     my $padding = (1024 - $size % 1024) % 1024;
@@ -342,7 +374,7 @@ sub zfs_create_zvol {
 
     push @$cmd, '-b', $scfg->{blocksize} if $scfg->{blocksize};
 
-    push @$cmd, '-V', "${size}k", "$scfg->{pool}/$zvol";
+    push @$cmd, '-V', "${size}k", "$pool/$zvol";
 
     $class->zfs_request($scfg, undef, @$cmd);
 }
@@ -350,7 +382,8 @@ sub zfs_create_zvol {
 sub zfs_create_subvol {
     my ($class, $scfg, $volname, $size) = @_;
 
-    my $dataset = "$scfg->{pool}/$volname";
+    my $pool = $get_local_pool->($scfg);
+    my $dataset = "$pool/$volname";
     my $quota = $size ? "${size}k" : "none";
 
     my $cmd =
@@ -362,11 +395,12 @@ sub zfs_create_subvol {
 sub zfs_delete_zvol {
     my ($class, $scfg, $zvol) = @_;
 
+    my $pool = $get_local_pool->($scfg);
     my $err;
 
     for (my $i = 0; $i < 6; $i++) {
 
-        eval { $class->zfs_request($scfg, undef, 'destroy', '-r', "$scfg->{pool}/$zvol"); };
+        eval { $class->zfs_request($scfg, undef, 'destroy', '-r', "$pool/$zvol"); };
         if ($err = $@) {
             if ($err =~ m/dataset is busy/) {
                 sleep(1);
@@ -387,6 +421,8 @@ sub zfs_delete_zvol {
 sub zfs_list_zvol {
     my ($class, $scfg) = @_;
 
+    my $pool = $get_local_pool->($scfg);
+
     my $text = $class->zfs_request(
         $scfg,
         10,
@@ -397,18 +433,18 @@ sub zfs_list_zvol {
         'volume,filesystem',
         '-d1',
         '-Hp',
-        $scfg->{pool},
+        $pool,
     );
     # It's still required to have zfs_parse_zvol_list filter by pool, because -d1 lists
-    # $scfg->{pool} too and while unlikely, it could be named to be mistaken for a volume.
-    my $zvols = zfs_parse_zvol_list($text, $scfg->{pool});
+    # $pool too and while unlikely, it could be named to be mistaken for a volume.
+    my $zvols = zfs_parse_zvol_list($text, $pool);
     return {} if !$zvols;
 
     my $list = {};
     foreach my $zvol (@$zvols) {
         my $name = $zvol->{name};
         my $parent = $zvol->{origin};
-        if ($zvol->{origin} && $zvol->{origin} =~ m/^$scfg->{pool}\/(\S+)$/) {
+        if ($zvol->{origin} && $zvol->{origin} =~ m/^$pool\/(\S+)$/) {
             $parent = $1;
         }
 
@@ -430,7 +466,8 @@ sub zfs_get_sorted_snapshot_list {
     my @params = ('-H', '-r', '-t', 'snapshot', '-o', 'name', $sort_params->@*);
 
     my $vname = ($class->parse_volname($volname))[1];
-    push @params, "$scfg->{pool}\/$vname";
+    my $pool = $get_local_pool->($scfg);
+    push @params, "$pool\/$vname";
 
     my $text = $class->zfs_request($scfg, undef, 'list', @params);
     my @snapshots = split(/\n/, $text);
@@ -466,9 +503,9 @@ sub volume_size_info {
 
     my (undef, $vname, undef, $parent, undef, undef, $format) = $class->parse_volname($volname);
 
+    my $pool = $get_local_pool->($scfg);
     my $attr = $format eq 'subvol' ? 'refquota' : 'volsize';
-    my ($size, $used) =
-        $class->zfs_get_properties($scfg, "$attr,usedbydataset", "$scfg->{pool}/$vname");
+    my ($size, $used) = $class->zfs_get_properties($scfg, "$attr,usedbydataset", "$pool/$vname");
 
     $used = ($used =~ /^(\d+)$/) ? $1 : 0;
 
@@ -483,7 +520,8 @@ sub volume_snapshot {
     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 
     my (undef, $vname, undef, undef, undef, undef, $format) = $class->parse_volname($volname);
-    my $snapshot_name = "$scfg->{pool}/$vname\@$snap";
+    my $pool = $get_local_pool->($scfg);
+    my $snapshot_name = "$pool/$vname\@$snap";
 
     $class->zfs_request($scfg, undef, 'snapshot', $snapshot_name);
 
@@ -491,7 +529,7 @@ sub volume_snapshot {
     # does not track this property for snapshosts and consequently does not roll
     # it back. so track this information manually.
     if ($format eq 'subvol') {
-        my $refquota = $class->zfs_get_properties($scfg, 'refquota', "$scfg->{pool}/$vname");
+        my $refquota = $class->zfs_get_properties($scfg, 'refquota', "$pool/$vname");
 
         $class->zfs_request(
             $scfg,
@@ -507,16 +545,18 @@ sub volume_snapshot_delete {
     my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 
     my $vname = ($class->parse_volname($volname))[1];
+    my $pool = $get_local_pool->($scfg);
 
     $class->deactivate_volume($storeid, $scfg, $vname, $snap, {});
-    $class->zfs_request($scfg, undef, 'destroy', "$scfg->{pool}/$vname\@$snap");
+    $class->zfs_request($scfg, undef, 'destroy', "$pool/$vname\@$snap");
 }
 
 sub volume_snapshot_rollback {
     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 
     my (undef, $vname, undef, undef, undef, undef, $format) = $class->parse_volname($volname);
-    my $snapshot_name = "$scfg->{pool}/$vname\@$snap";
+    my $pool = $get_local_pool->($scfg);
+    my $snapshot_name = "$pool/$vname\@$snap";
 
     my $msg = $class->zfs_request($scfg, undef, 'rollback', $snapshot_name);
 
@@ -527,7 +567,7 @@ sub volume_snapshot_rollback {
 
         if ($refquota =~ m/^\d+$/) {
             $class->zfs_request(
-                $scfg, undef, 'set', "refquota=${refquota}", "$scfg->{pool}/$vname",
+                $scfg, undef, 'set', "refquota=${refquota}", "$pool/$vname",
             );
         } elsif ($refquota ne "-") {
             # refquota user property was set, but not a number -> warn
@@ -539,7 +579,7 @@ sub volume_snapshot_rollback {
     # caches, they get mounted in activate volume again
     # see zfs bug #10931 https://github.com/openzfs/zfs/issues/10931
     if ($format eq 'subvol') {
-        eval { $class->zfs_request($scfg, undef, 'unmount', "$scfg->{pool}/$vname"); };
+        eval { $class->zfs_request($scfg, undef, 'unmount', "$pool/$vname"); };
         if (my $err = $@) {
             die $err if $err !~ m/not currently mounted$/;
         }
@@ -582,7 +622,8 @@ sub volume_snapshot_info {
     my @params = ('-Hp', '-r', '-t', 'snapshot', '-o', 'name,guid,creation');
 
     my $vname = ($class->parse_volname($volname))[1];
-    push @params, "$scfg->{pool}\/$vname";
+    my $pool = $get_local_pool->($scfg);
+    push @params, "$pool\/$vname";
 
     my $text = $class->zfs_request($scfg, undef, 'list', @params);
     my @lines = split(/\n/, $text);
@@ -619,7 +660,7 @@ sub activate_storage {
     my ($class, $storeid, $scfg, $cache) = @_;
 
     # Note: $scfg->{pool} can include dataset <pool>/<dataset>
-    my $dataset = $scfg->{pool};
+    my $dataset = $get_local_pool->($scfg);
     my $pool = ($dataset =~ s!/.*$!!r);
 
     return 1 if dataset_mounted_heuristic($dataset); # early return
@@ -657,13 +698,14 @@ sub activate_volume {
     return 1 if defined($snapname);
 
     my (undef, $dataset, undef, undef, undef, undef, $format) = $class->parse_volname($volname);
+    my $pool = $get_local_pool->($scfg);
 
     if ($format eq 'raw') {
         $class->zfs_wait_for_zvol_link($scfg, $volname);
     } elsif ($format eq 'subvol') {
-        my $mounted = $class->zfs_get_properties($scfg, 'mounted', "$scfg->{pool}/$dataset");
+        my $mounted = $class->zfs_get_properties($scfg, 'mounted', "$pool/$dataset");
         if ($mounted !~ m/^yes$/) {
-            $class->zfs_request($scfg, undef, 'mount', "$scfg->{pool}/$dataset");
+            $class->zfs_request($scfg, undef, 'mount', "$pool/$dataset");
         }
     }
 
@@ -686,28 +728,25 @@ sub clone_image {
     die "clone_image only works on base images\n" if !$isBase;
 
     my $name = $class->find_free_diskname($storeid, $scfg, $vmid, $format);
+    my $pool = $get_local_pool->($scfg);
 
     if ($format eq 'subvol') {
         my $size = $class->zfs_request(
-            $scfg, undef, 'list', '-Hp', '-o', 'refquota', "$scfg->{pool}/$basename",
+            $scfg, undef, 'list', '-Hp', '-o', 'refquota', "$pool/$basename",
         );
         chomp($size);
         $class->zfs_request(
             $scfg,
             undef,
             'clone',
-            "$scfg->{pool}/$basename\@$snap",
-            "$scfg->{pool}/$name",
+            "$pool/$basename\@$snap",
+            "$pool/$name",
             '-o',
             "refquota=$size",
         );
     } else {
         $class->zfs_request(
-            $scfg,
-            undef,
-            'clone',
-            "$scfg->{pool}/$basename\@$snap",
-            "$scfg->{pool}/$name",
+            $scfg, undef, 'clone', "$pool/$basename\@$snap", "$pool/$name",
         );
     }
 
@@ -731,8 +770,9 @@ sub create_base {
         $newname =~ s/^vm-/base-/;
     }
     my $newvolname = $basename ? "$basename/$newname" : "$newname";
+    my $pool = $get_local_pool->($scfg);
 
-    $class->zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
+    $class->zfs_request($scfg, undef, 'rename', "$pool/$name", "$pool/$newname");
 
     my $running = undef; #fixme : is create_base always offline ?
 
@@ -756,7 +796,8 @@ sub volume_resize {
         $new_size = $new_size + $padding;
     }
 
-    $class->zfs_request($scfg, undef, 'set', "$attr=${new_size}k", "$scfg->{pool}/$vname");
+    my $pool = $get_local_pool->($scfg);
+    $class->zfs_request($scfg, undef, 'set', "$attr=${new_size}k", "$pool/$vname");
 
     return $new_size;
 }
@@ -822,7 +863,8 @@ sub volume_export {
         my $arg = $with_snapshots ? '-I' : '-i';
         push @$cmd, $arg, $base_snapshot;
     }
-    push @$cmd, '--', "$scfg->{pool}/$dataset\@$snapshot";
+    my $pool = $get_local_pool->($scfg);
+    push @$cmd, '--', "$pool/$dataset\@$snapshot";
 
     run_command($cmd, output => $fd);
 
@@ -862,8 +904,9 @@ sub volume_import {
 
     my (undef, $dataset, $vmid, undef, undef, undef, $volume_format) =
         $class->parse_volname($volname);
+    my $pool = $get_local_pool->($scfg);
 
-    my $zfspath = "$scfg->{pool}/$dataset";
+    my $zfspath = "$pool/$dataset";
     my $suffix = defined($base_snapshot) ? "\@$base_snapshot" : '';
     my $exists = 0 == run_command(
         ['zfs', 'get', '-H', 'name', $zfspath . $suffix],
@@ -876,7 +919,7 @@ sub volume_import {
         die "volume '$zfspath' already exists\n" if !$allow_rename;
         warn "volume '$zfspath' already exists - importing with a different name\n";
         $dataset = $class->find_free_diskname($storeid, $scfg, $vmid, $volume_format);
-        $zfspath = "$scfg->{pool}/$dataset";
+        $zfspath = "$pool/$dataset";
     }
 
     eval { run_command(['zfs', 'recv', '-F', '--', $zfspath], input => "<&$fd") };
@@ -909,7 +952,7 @@ sub rename_volume {
     $target_volname = $class->find_free_diskname($storeid, $scfg, $target_vmid, $format)
         if !$target_volname;
 
-    my $pool = $scfg->{pool};
+    my $pool = $get_local_pool->($scfg);
     my $source_zfspath = "${pool}/${source_image}";
     my $target_zfspath = "${pool}/${target_volname}";
 
@@ -933,4 +976,16 @@ sub rename_snapshot {
     die "rename_snapshot is not supported for $class";
 }
 
+sub check_config {
+    my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_;
+
+    my $checked = $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck);
+
+    # check if either target or mapping is set
+    die "zfspool storage '$sectionId' has neither 'pool' nor 'mapping' defined\n"
+        if !defined($checked->{pool}) && !defined($checked->{mapping});
+
+    return $checked;
+}
+
 1;
-- 
2.47.3