[pve-devel] [PATCH storage v8 5/9] introduce $vtype+meta export formats

[Filip Schauer <f.schauer@proxmox.com>]

These new export formats include a JSON metadata header containing the
vtype and the format. This allows for future extensibility without
breaking backward compatibility when adding additional metadata.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 src/PVE/Storage.pm        |  16 +++++-
 src/PVE/Storage/Plugin.pm | 101 +++++++++++++++++++++++++++++++++-----
 2 files changed, 103 insertions(+), 14 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 3467a90..d380160 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -47,7 +47,21 @@ use constant APIVER => 12;
 # see https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
 use constant APIAGE => 3;
 
-our $KNOWN_EXPORT_FORMATS = ['raw+size', 'tar+size', 'qcow2+size', 'vmdk+size', 'zfs', 'btrfs'];
+our $KNOWN_EXPORT_FORMATS = [
+    'raw+size',
+    'tar+size',
+    'qcow2+size',
+    'vmdk+size',
+    'zfs',
+    'btrfs',
+    'images+meta',
+    'rootdir+meta',
+    'iso+meta',
+    'vztmpl+meta',
+    'backup+meta',
+    'snippets+meta',
+    'import+meta',
+];
 
 # load standard plugins
 PVE::Storage::DirPlugin->register();
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index ecf68c8..29628ed 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -2022,6 +2022,25 @@ sub read_common_header($) {
     return $size;
 }
 
+sub write_meta_header($$) {
+    my ($fh, $meta) = @_;
+    my $meta_json = JSON::encode_json($meta);
+    $meta_json = Encode::encode('utf8', $meta_json);
+    syswrite($fh, pack("Q<", length($meta_json)), 8);
+    syswrite($fh, $meta_json, length($meta_json));
+}
+
+sub read_meta_header($) {
+    my ($fh) = @_;
+    sysread($fh, my $meta_size, 8);
+    $meta_size = unpack('Q<', $meta_size);
+    die "import: no meta size found in export header, aborting.\n" if !defined($meta_size);
+    sysread($fh, my $meta_json, $meta_size);
+    $meta_json = Encode::decode('utf8', $meta_json);
+    my $meta = JSON::decode_json($meta_json);
+    return $meta;
+}
+
 # Export a volume into a file handle as a stream of desired format.
 sub volume_export {
     my ($class, $scfg, $storeid, $fh, $volname, $format, $snapshot, $base_snapshot, $with_snapshots)
@@ -2033,7 +2052,7 @@ sub volume_export {
     my $err_msg = "volume export format $format not available for $class\n";
     if ($scfg->{path} && !defined($snapshot) && !defined($base_snapshot)) {
         my ($file) = $class->path($scfg, $volname, $storeid) or die $err_msg;
-        my $file_format = ($class->parse_volname($volname))[6];
+        my ($vtype, $file_format) = ($class->parse_volname($volname))[0, 6];
         my $size = file_size_info($file, undef, $file_format);
 
         if ($format eq 'raw+size') {
@@ -2077,6 +2096,52 @@ sub volume_export {
                 output => '>&' . fileno($fh),
             );
             return;
+        } elsif ($format eq "$vtype+meta") {
+            die "format $file_format cannot be exported without snapshots\n"
+                if !$with_snapshots && $file_format =~ /^qcow2|vmdk$/;
+            die "format $file_format cannot be exported with snapshots\n"
+                if $with_snapshots && $file_format =~ /^raw|subvol$/;
+
+            my $data_format;
+            if ($file_format eq 'subvol') {
+                $data_format = 'tar';
+            } else {
+                $data_format = $file_format;
+            }
+
+            my $meta = {
+                vtype => $vtype,
+                format => $data_format,
+            };
+
+            write_meta_header($fh, $meta);
+            write_common_header($fh, $size);
+            if ($data_format =~ /^(raw|ova|ovf|qcow2|vmdk)$/) {
+                run_command(
+                    ['dd', "if=$file", "bs=4k", "status=progress"],
+                    output => '>&' . fileno($fh),
+                );
+            } elsif ($data_format eq 'tar') {
+                run_command(
+                    ['tar', @COMMON_TAR_FLAGS, '-cf', '-', '-C', $file, '.'],
+                    output => '>&' . fileno($fh),
+                );
+            } else {
+                run_command(
+                    [
+                        'qemu-img',
+                        'convert',
+                        '-f',
+                        $file_format,
+                        '-O',
+                        'raw',
+                        $file,
+                        '/dev/stdout',
+                    ],
+                    output => '>&' . fileno($fh),
+                );
+            }
+            return;
         }
     }
     die $err_msg;
@@ -2088,11 +2153,11 @@ sub volume_export_formats {
         my ($vtype, $format) = ($class->parse_volname($volname))[0, 6];
 
         if ($with_snapshots) {
-            return ($format . '+size') if ($format eq 'qcow2' || $format eq 'vmdk');
+            return ("$vtype+meta", $format . '+size') if $format =~ /^(qcow2|vmdk)$/;
             return ();
         }
-        return ('tar+size') if $format eq 'subvol';
-        return ('raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
+        return ("$vtype+meta", 'tar+size') if $format eq 'subvol';
+        return ("$vtype+meta", 'raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
     }
     return ();
 }
@@ -2112,18 +2177,28 @@ sub volume_import {
         $allow_rename,
     ) = @_;
 
-    die "volume import format '$format' not available for $class\n"
-        if $format !~ /^(raw|tar|qcow2|vmdk)\+size$/;
-    my $data_format = $1;
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $file_format) =
+        $class->parse_volname($volname);
+
+    my $meta;
+    my $data_format;
+
+    if ($format =~ /^(images|rootdir|iso|vztmpl|backup|snippets|import)\+meta$/) {
+        die "volume type does not match import format\n" if $1 ne $vtype;
+        $meta = read_meta_header($fh);
+        die "volume type does not match metadata header\n" if $meta->{vtype} ne $vtype;
+        $data_format = $meta->{format};
+    } elsif ($format =~ /^(raw|tar|qcow2|vmdk)\+size$/) {
+        $data_format = $1;
+    } else {
+        die "volume import format '$format' not available for $class\n";
+    }
 
     die "format $format cannot be imported without snapshots\n"
         if !$with_snapshots && ($data_format eq 'qcow2' || $data_format eq 'vmdk');
     die "format $format cannot be imported with snapshots\n"
         if $with_snapshots && ($data_format eq 'raw' || $data_format eq 'tar');
 
-    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $file_format) =
-        $class->parse_volname($volname);
-
     # XXX: Should we bother with conversion routines at this level? This won't
     # happen without manual CLI usage, so for now we just error out...
     if (
@@ -2201,11 +2276,11 @@ sub volume_import_formats {
         my ($vtype, $format) = ($class->parse_volname($volname))[0, 6];
 
         if ($with_snapshots) {
-            return ($format . '+size') if ($format eq 'qcow2' || $format eq 'vmdk');
+            return ("$vtype+meta", $format . '+size') if $format =~ /^(qcow2|vmdk)$/;
             return ();
         }
-        return ('tar+size') if $format eq 'subvol';
-        return ('raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
+        return ("$vtype+meta", 'tar+size') if $format eq 'subvol';
+        return ("$vtype+meta", 'raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
     }
     return ();
 }
-- 
2.47.3



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel