From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pve-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
by lore.proxmox.com (Postfix) with ESMTPS id 1DB7B1FF15E
for <inbox@lore.proxmox.com>; Tue, 11 Mar 2025 15:24:58 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id B8FB012576;
Tue, 11 Mar 2025 15:24:11 +0100 (CET)
From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Date: Tue, 11 Mar 2025 15:23:24 +0100
Message-Id: <20250311142328.112538-6-f.schauer@proxmox.com>
X-Mailer: git-send-email 2.39.5
In-Reply-To: <20250311142328.112538-1-f.schauer@proxmox.com>
References: <20250311142328.112538-1-f.schauer@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL -0.018 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
KAM_SHORT 0.001 Use of a URL Shortener for very short URL
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pve-devel] [PATCH storage v7 5/9] introduce $vtype+meta export
formats
X-BeenThere: pve-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox VE development discussion <pve-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pve-devel>,
<mailto:pve-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pve-devel/>
List-Post: <mailto:pve-devel@lists.proxmox.com>
List-Help: <mailto:pve-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel>,
<mailto:pve-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox VE development discussion <pve-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pve-devel-bounces@lists.proxmox.com
Sender: "pve-devel" <pve-devel-bounces@lists.proxmox.com>
These new export formats include a JSON metadata header containing the
vtype and the format. This allows for future extensibility without
breaking backward compatibility when adding additional metadata.
Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
src/PVE/Storage.pm | 16 ++++++-
src/PVE/Storage/Plugin.pm | 94 +++++++++++++++++++++++++++++++++------
2 files changed, 95 insertions(+), 15 deletions(-)
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 88e4e6d..cfadcc4 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -48,7 +48,21 @@ use constant APIVER => 10;
# see https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
use constant APIAGE => 1;
-our $KNOWN_EXPORT_FORMATS = ['raw+size', 'tar+size', 'qcow2+size', 'vmdk+size', 'zfs', 'btrfs'];
+our $KNOWN_EXPORT_FORMATS = [
+ 'raw+size',
+ 'tar+size',
+ 'qcow2+size',
+ 'vmdk+size',
+ 'zfs',
+ 'btrfs',
+ 'images+meta',
+ 'rootdir+meta',
+ 'iso+meta',
+ 'vztmpl+meta',
+ 'backup+meta',
+ 'snippets+meta',
+ 'import+meta'
+];
# load standard plugins
PVE::Storage::DirPlugin->register();
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index a97f449..34f26a3 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -1693,6 +1693,25 @@ sub read_common_header($) {
return $size;
}
+sub write_meta_header($$) {
+ my ($fh, $meta) = @_;
+ my $meta_json = JSON::encode_json($meta);
+ $meta_json = Encode::encode('utf8', $meta_json);
+ syswrite($fh, pack("Q<", length($meta_json)), 8);
+ syswrite($fh, $meta_json, length($meta_json));
+}
+
+sub read_meta_header($) {
+ my ($fh) = @_;
+ sysread($fh, my $meta_size, 8);
+ $meta_size = unpack('Q<', $meta_size);
+ die "import: no meta size found in export header, aborting.\n" if !defined($meta_size);
+ sysread($fh, my $meta_json, $meta_size);
+ $meta_json = Encode::decode('utf8', $meta_json);
+ my $meta = JSON::decode_json($meta_json);
+ return $meta;
+}
+
# Export a volume into a file handle as a stream of desired format.
sub volume_export {
my ($class, $scfg, $storeid, $fh, $volname, $format, $snapshot, $base_snapshot, $with_snapshots) = @_;
@@ -1700,7 +1719,7 @@ sub volume_export {
my $err_msg = "volume export format $format not available for $class\n";
if ($scfg->{path} && !defined($snapshot) && !defined($base_snapshot)) {
my ($file) = $class->path($scfg, $volname, $storeid) or die $err_msg;
- my $file_format = ($class->parse_volname($volname))[6];
+ my ($vtype, $file_format) = ($class->parse_volname($volname))[0, 6];
my $size = file_size_info($file, undef, $file_format);
if ($format eq 'raw+size') {
@@ -1723,7 +1742,44 @@ sub volume_export {
die $err_msg if $file_format ne 'subvol';
write_common_header($fh, $size);
run_command(['tar', @COMMON_TAR_FLAGS, '-cf', '-', '-C', $file, '.'],
- output => '>&'.fileno($fh));
+ output => '>&'.fileno($fh));
+ return;
+ } elsif ($format eq "$vtype+meta") {
+ die "format $file_format cannot be exported without snapshots\n"
+ if !$with_snapshots && $file_format =~ /^qcow2|vmdk$/;
+ die "format $file_format cannot be exported with snapshots\n"
+ if $with_snapshots && $file_format =~ /^raw|subvol$/;
+
+ my $data_format;
+ if ($file_format eq 'subvol') {
+ $data_format = 'tar';
+ } else {
+ $data_format = $file_format;
+ }
+
+ my $meta = {
+ vtype => $vtype,
+ format => $data_format,
+ };
+
+ write_meta_header($fh, $meta);
+ write_common_header($fh, $size);
+ if ($data_format =~ /^(raw|ova|ovf|qcow2|vmdk)$/) {
+ run_command(
+ ['dd', "if=$file", "bs=4k", "status=progress"],
+ output => '>&'.fileno($fh)
+ );
+ } elsif ($data_format eq 'tar') {
+ run_command(
+ ['tar', @COMMON_TAR_FLAGS, '-cf', '-', '-C', $file, '.'],
+ output => '>&'.fileno($fh)
+ );
+ } else {
+ run_command(
+ ['qemu-img', 'convert', '-f', $file_format, '-O', 'raw', $file, '/dev/stdout'],
+ output => '>&'.fileno($fh)
+ );
+ }
return;
}
}
@@ -1738,11 +1794,11 @@ sub volume_export_formats {
my ($vtype, $format) = ($class->parse_volname($volname))[0, 6];
if ($with_snapshots) {
- return ($format.'+size') if ($format eq 'qcow2' || $format eq 'vmdk');
+ return ("$vtype+meta", $format.'+size') if $format =~ /^(qcow2|vmdk)$/;
return ();
}
- return ('tar+size') if $format eq 'subvol';
- return ('raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
+ return ("$vtype+meta", 'tar+size') if $format eq 'subvol';
+ return ("$vtype+meta", 'raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
}
return ();
}
@@ -1751,18 +1807,28 @@ sub volume_export_formats {
sub volume_import {
my ($class, $scfg, $storeid, $fh, $volname, $format, $snapshot, $base_snapshot, $with_snapshots, $allow_rename) = @_;
- die "volume import format '$format' not available for $class\n"
- if $format !~ /^(raw|tar|qcow2|vmdk)\+size$/;
- my $data_format = $1;
+ my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $file_format) =
+ $class->parse_volname($volname);
+
+ my $meta;
+ my $data_format;
+
+ if ($format =~ /^(images|rootdir|iso|vztmpl|backup|snippets|import)\+meta$/) {
+ die "volume type does not match import format\n" if $1 ne $vtype;
+ $meta = read_meta_header($fh);
+ die "volume type does not match metadata header\n" if $meta->{vtype} ne $vtype;
+ $data_format = $meta->{format};
+ } elsif ($format =~ /^(raw|tar|qcow2|vmdk)\+size$/) {
+ $data_format = $1;
+ } else {
+ die "volume import format '$format' not available for $class\n";
+ }
die "format $format cannot be imported without snapshots\n"
if !$with_snapshots && ($data_format eq 'qcow2' || $data_format eq 'vmdk');
die "format $format cannot be imported with snapshots\n"
if $with_snapshots && ($data_format eq 'raw' || $data_format eq 'tar');
- my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $file_format) =
- $class->parse_volname($volname);
-
# XXX: Should we bother with conversion routines at this level? This won't
# happen without manual CLI usage, so for now we just error out...
if (
@@ -1835,11 +1901,11 @@ sub volume_import_formats {
my ($vtype, $format) = ($class->parse_volname($volname))[0, 6];
if ($with_snapshots) {
- return ($format.'+size') if ($format eq 'qcow2' || $format eq 'vmdk');
+ return ("$vtype+meta", $format.'+size') if $format =~ /^(qcow2|vmdk)$/;
return ();
}
- return ('tar+size') if $format eq 'subvol';
- return ('raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
+ return ("$vtype+meta", 'tar+size') if $format eq 'subvol';
+ return ("$vtype+meta", 'raw+size') if $vtype =~ /^(iso|snippets|vztmpl|import)$/;
}
return ();
}
--
2.39.5
_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel