* [pve-devel] [RFC v2 pve-container pve-manager 1/3] backup: do not delete not backed-up mps on restore
2023-10-23 11:18 [pve-devel] [RFC v2 pve-container pve-manager 0/3] add partial restore Christian Ebner
@ 2023-10-23 11:18 ` Christian Ebner
2023-10-23 11:18 ` [pve-devel] [RFC v2 pve-container pve-manager 2/3] api: allow to exclude mountpoins from restore Christian Ebner
` (2 subsequent siblings)
3 siblings, 0 replies; 6+ messages in thread
From: Christian Ebner @ 2023-10-23 11:18 UTC (permalink / raw)
To: pve-devel
The current behaviour of the restore is to recreate all backed up
mountpoints and remove all previous ones, causing potential data loss on
restore when the mountpoint was not included in the backup and the user
not aware of this behaviour.
By checking the mountpoint configuration from the backup, only recreate
the disks which are included in the backup and add them as mountpoints.
Leave all other mountpoints untouched and attach them as unused disks
as final step of the restore.
To facilitate selective restore from PBS backups, split the currently
single root pxar archive into a pxar archive for each individual
mountpoint, while remaining backwards compatible.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since v1:
- deep clone of orig_mp_param, otherwise the variable points to the same
data.
- refactor restore_archive params
src/PVE/API2/LXC.pm | 34 +++++++++++++++-----
src/PVE/LXC/Create.pm | 72 +++++++++++++++++++++++++++++++++++--------
src/PVE/VZDump/LXC.pm | 10 +++---
3 files changed, 92 insertions(+), 24 deletions(-)
diff --git a/src/PVE/API2/LXC.pm b/src/PVE/API2/LXC.pm
index 28d14de..090dddf 100644
--- a/src/PVE/API2/LXC.pm
+++ b/src/PVE/API2/LXC.pm
@@ -4,6 +4,7 @@ use strict;
use warnings;
use Socket qw(SOCK_STREAM);
+use Storable qw(dclone);
use PVE::SafeSyslog;
use PVE::Tools qw(extract_param run_command);
@@ -381,8 +382,10 @@ __PACKAGE__->register_method({
my $vollist = [];
eval {
my $orig_mp_param; # only used if $restore
+ my $clear_mps;
if ($restore) {
die "can't overwrite running container\n" if PVE::LXC::check_running($vmid);
+
if ($archive ne '-') {
my $orig_conf;
print "recovering backed-up configuration from '$archive'\n";
@@ -419,11 +422,19 @@ __PACKAGE__->register_method({
print "recovering backed-up configuration from '$archive'\n";
(undef, $orig_mp_param) = PVE::LXC::Create::recover_config($storage_cfg, $archive, $vmid);
}
- $mp_param = $orig_mp_param;
+ $mp_param = dclone $orig_mp_param;
die "rootfs configuration could not be recovered, please check and specify manually!\n"
if !defined($mp_param->{rootfs});
PVE::LXC::Config->foreach_volume($mp_param, sub {
my ($ms, $mountpoint) = @_;
+ if ($ms eq 'rootfs' || $mountpoint->{backup}) {
+ # backup conf contains the mp, clear for retsore
+ $clear_mps->{$ms} = $mountpoint;
+ } else {
+ # do not add as mp, will be attach as unused at the end
+ delete $mp_param->{$ms};
+ return;
+ }
my $type = $mountpoint->{type};
if ($type eq 'volume') {
die "unable to detect disk size - please specify $ms (size)\n"
@@ -459,18 +470,24 @@ __PACKAGE__->register_method({
$vollist = PVE::LXC::create_disks($storage_cfg, $vmid, $mp_param, $conf);
- # we always have the 'create' lock so check for more than 1 entry
- if (scalar(keys %$old_conf) > 1) {
- # destroy old container volumes
- PVE::LXC::destroy_lxc_container($storage_cfg, $vmid, $old_conf, { lock => 'create' });
- }
+ # Delete old mountpoints which are restored from backup.
+ PVE::LXC::Config->foreach_volume($old_conf, sub {
+ my ($name, $mountpoint, undef) = @_;
+ return if !defined($clear_mps->{$name});
+ PVE::LXC::delete_mountpoint_volume($storage_cfg, $vmid, $mountpoint->{volume});
+ });
eval {
my $rootdir = PVE::LXC::mount_all($vmid, $storage_cfg, $conf, 1);
$bwlimit = PVE::Storage::get_bandwidth_limit('restore', [keys %used_storages], $bwlimit);
print "restoring '$archive' now..\n"
if $restore && $archive ne '-';
- PVE::LXC::Create::restore_archive($storage_cfg, $archive, $rootdir, $conf, $ignore_unpack_errors, $bwlimit);
+
+ my $restore_opts = {
+ 'orig_mps' => $orig_mp_param,
+ };
+ PVE::LXC::Create::restore_archive(
+ $storage_cfg, $archive, $rootdir, $conf, $ignore_unpack_errors, $bwlimit, $restore_opts);
if ($restore) {
print "merging backed-up and given configuration..\n";
@@ -501,6 +518,9 @@ __PACKAGE__->register_method({
$conf->{template} = 1;
}
PVE::LXC::Config->write_config($vmid, $conf);
+
+ # Attach all additionally found mountpoints as unused disks.
+ PVE::LXC::rescan($vmid, 1, 0);
};
if (my $err = $@) {
PVE::LXC::destroy_disks($storage_cfg, $vollist);
diff --git a/src/PVE/LXC/Create.pm b/src/PVE/LXC/Create.pm
index f4c3220..98ab4a4 100644
--- a/src/PVE/LXC/Create.pm
+++ b/src/PVE/LXC/Create.pm
@@ -83,27 +83,33 @@ sub detect_architecture {
}
sub restore_archive {
- my ($storage_cfg, $archive, $rootdir, $conf, $no_unpack_error, $bwlimit) = @_;
+ my ($storage_cfg, $archive, $rootdir, $conf, $no_unpack_error, $bwlimit, $restore_opts) = @_;
+
+ PVE::LXC::Config->foreach_volume($restore_opts->{orig_mps}, sub {
+ my ($name, $vol, undef) = @_;
+ $restore_opts->{orig_mps}->{$name} = $vol;
+ });
my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive, 1);
if (defined($storeid)) {
my $scfg = PVE::Storage::storage_check_enabled($storage_cfg, $storeid);
if ($scfg->{type} eq 'pbs') {
- return restore_proxmox_backup_archive($storage_cfg, $archive, $rootdir, $conf, $no_unpack_error, $bwlimit);
+ return restore_proxmox_backup_archive(
+ $storage_cfg, $archive, $rootdir, $conf, $no_unpack_error, $bwlimit, $restore_opts);
}
}
$archive = PVE::Storage::abs_filesystem_path($storage_cfg, $archive) if $archive ne '-';
- restore_tar_archive($archive, $rootdir, $conf, $no_unpack_error, $bwlimit);
+ restore_tar_archive($archive, $rootdir, $conf, $no_unpack_error, $bwlimit, $restore_opts);
}
sub restore_proxmox_backup_archive {
- my ($storage_cfg, $archive, $rootdir, $conf, $no_unpack_error, $bwlimit) = @_;
+ my ($storage_cfg, $archive, $rootdir, $conf, $no_unpack_error, $bwlimit, $restore_opts) = @_;
my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
my $scfg = PVE::Storage::storage_config($storage_cfg, $storeid);
- my ($vtype, $name, undef, undef, undef, undef, $format) =
+ my ($vtype, $snapshot, undef, undef, undef, undef, $format) =
PVE::Storage::parse_volname($storage_cfg, $archive);
die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
@@ -114,14 +120,47 @@ sub restore_proxmox_backup_archive {
my $userns_cmd = PVE::LXC::userns_command($id_map);
my $cmd = "restore";
- my $param = [$name, "root.pxar", $rootdir, '--allow-existing-dirs'];
+ PVE::LXC::Config->foreach_volume($conf, sub {
+ my ($name, $vol, undef) = @_;
- if ($no_unpack_error) {
- push(@$param, '--ignore-extract-device-errors');
- }
+ my $orig_mp = $restore_opts->{orig_mps}->{$name};
+ if (!defined($orig_mp)) {
+ print "'$name': mp not present in original backup.\n";
+ return;
+ }
- PVE::Storage::PBSPlugin::run_raw_client_cmd(
- $scfg, $storeid, $cmd, $param, userns_cmd => $userns_cmd);
+ if ($name ne 'rootfs' && (!defined($orig_mp->{backup}) || !$orig_mp->{backup})) {
+ print "'$name': mp not included in original backup.\n";
+ return;
+ }
+
+ $name = 'root' if $name eq 'rootfs';
+ my $target = "$rootdir/.$vol->{mp}";
+ if ($orig_mp->{mp} ne $vol->{mp}) {
+ print "'$name': path differs from backed-up path, restore to backed-up path.\n";
+ print " If this is not intended, change the path after restore.\n";
+ $target = "$rootdir/.$orig_mp->{mp}";
+ }
+
+ my $param = [$snapshot, "$name.pxar", $target, '--allow-existing-dirs'];
+
+ if ($no_unpack_error) {
+ push(@$param, '--ignore-extract-device-errors');
+ }
+
+ eval {
+ # This will fail for backups created without mp archive splitting
+ PVE::Storage::PBSPlugin::run_raw_client_cmd(
+ $scfg, $storeid, $cmd, $param, userns_cmd => $userns_cmd);
+ };
+ my $err = $@;
+ if ($err) {
+ # Only handle root restore failure as hard error
+ die $err if $name eq 'root';
+ print "extracting moutpoint '$name' failed:\n$err";
+ print "backup created with older version?\n";
+ }
+ });
# if arch is set, we do not try to autodetect it
return if defined($conf->{arch});
@@ -130,11 +169,20 @@ sub restore_proxmox_backup_archive {
}
sub restore_tar_archive {
- my ($archive, $rootdir, $conf, $no_unpack_error, $bwlimit) = @_;
+ my ($archive, $rootdir, $conf, $no_unpack_error, $bwlimit, $restore_opts) = @_;
my ($id_map, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
my $userns_cmd = PVE::LXC::userns_command($id_map);
+ PVE::LXC::Config->foreach_volume($conf, sub {
+ my ($name, $vol, undef) = @_;
+ my $orig_mp = $restore_opts->{orig_mps}->{$name};
+ if (defined($orig_mp->{mp}) && $orig_mp->{mp} ne $vol->{mp}) {
+ print "'$name': path differs from backed-up path, restore to backed-up path.\n";
+ print " If this is not intended, change the path after restore.\n";
+ }
+ });
+
my $archive_fh;
my $tar_input = '<&STDIN';
my @compression_opt;
diff --git a/src/PVE/VZDump/LXC.pm b/src/PVE/VZDump/LXC.pm
index c68a06f..0d411b8 100644
--- a/src/PVE/VZDump/LXC.pm
+++ b/src/PVE/VZDump/LXC.pm
@@ -381,11 +381,11 @@ sub archive {
push @$param, "fw.conf:$fw_conf";
}
- my $rootdir = $snapdir;
- push @$param, "root.pxar:$rootdir";
-
- foreach my $disk (@sources) {
- push @$param, '--include-dev', "$snapdir/$disk";
+ foreach my $disk (@$disks) {
+ my $name = $disk->{name};
+ # Needed for backwards compatibility with previous backups
+ $name = 'root' if $name eq 'rootfs';
+ push @$param, "$name.pxar:$snapdir/.$disk->{mp}";
}
push @$param, '--skip-lost-and-found' if $userns_cmd;
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread* [pve-devel] [RFC v2 pve-container pve-manager 2/3] api: allow to exclude mountpoins from restore
2023-10-23 11:18 [pve-devel] [RFC v2 pve-container pve-manager 0/3] add partial restore Christian Ebner
2023-10-23 11:18 ` [pve-devel] [RFC v2 pve-container pve-manager 1/3] backup: do not delete not backed-up mps on restore Christian Ebner
@ 2023-10-23 11:18 ` Christian Ebner
2023-10-23 11:18 ` [pve-devel] [RFC v2 pve-container pve-manager 3/3] ui: lxc restore: add selective mountpoint restore Christian Ebner
2023-10-23 12:47 ` [pve-devel] [RFC v2 pve-container pve-manager 0/3] add partial restore Fiona Ebner
3 siblings, 0 replies; 6+ messages in thread
From: Christian Ebner @ 2023-10-23 11:18 UTC (permalink / raw)
To: pve-devel
Adds an optional parameter which allows to specify the names of
mountpoints which should be excluded from restore.
These mountpoints are not (re-)created and files located below the
mountpoint are not restored from backup.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since v1:
not present in v1
src/PVE/API2/LXC.pm | 14 ++++++++++++--
src/PVE/LXC/Create.pm | 11 +++++++++++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/src/PVE/API2/LXC.pm b/src/PVE/API2/LXC.pm
index 090dddf..523afb8 100644
--- a/src/PVE/API2/LXC.pm
+++ b/src/PVE/API2/LXC.pm
@@ -162,6 +162,12 @@ __PACKAGE__->register_method({
type => 'boolean',
description => "Mark this as restore task.",
},
+ 'exclude-mps' => {
+ optional => 1,
+ type => 'string',
+ description => "Comma separated lists of mountpoint to exclude from restore.",
+ pattern => '(mp\d+)(,mp\d+)*',
+ },
unique => {
optional => 1,
type => 'boolean',
@@ -222,6 +228,8 @@ __PACKAGE__->register_method({
# 'unprivileged' is read-only, so we can't pass it to update_pct_config
my $unprivileged = extract_param($param, 'unprivileged');
my $restore = extract_param($param, 'restore');
+ my $exclude_mps = extract_param($param, 'exclude-mps');
+ my %excludes = map {$_ => 1} split(',', $exclude_mps) if $exclude_mps;
my $unique = extract_param($param, 'unique');
$param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
@@ -427,8 +435,9 @@ __PACKAGE__->register_method({
if !defined($mp_param->{rootfs});
PVE::LXC::Config->foreach_volume($mp_param, sub {
my ($ms, $mountpoint) = @_;
- if ($ms eq 'rootfs' || $mountpoint->{backup}) {
- # backup conf contains the mp, clear for retsore
+ if ($ms eq 'rootfs' || ($mountpoint->{backup} && !$excludes{$ms})) {
+ # backup conf contains the mp and it is not in exclude list,
+ # clear for retsore
$clear_mps->{$ms} = $mountpoint;
} else {
# do not add as mp, will be attach as unused at the end
@@ -485,6 +494,7 @@ __PACKAGE__->register_method({
my $restore_opts = {
'orig_mps' => $orig_mp_param,
+ 'excludes' => \%excludes,
};
PVE::LXC::Create::restore_archive(
$storage_cfg, $archive, $rootdir, $conf, $ignore_unpack_errors, $bwlimit, $restore_opts);
diff --git a/src/PVE/LXC/Create.pm b/src/PVE/LXC/Create.pm
index 98ab4a4..34929d5 100644
--- a/src/PVE/LXC/Create.pm
+++ b/src/PVE/LXC/Create.pm
@@ -129,6 +129,11 @@ sub restore_proxmox_backup_archive {
return;
}
+ if (defined($restore_opts->{excludes}->{$name})) {
+ print "'$name': mp excluded from restore.\n";
+ return;
+ }
+
if ($name ne 'rootfs' && (!defined($orig_mp->{backup}) || !$orig_mp->{backup})) {
print "'$name': mp not included in original backup.\n";
return;
@@ -221,6 +226,12 @@ sub restore_tar_archive {
push @$cmd, '--skip-old-files';
push @$cmd, '--anchored';
push @$cmd, '--exclude' , './dev/*';
+
+ my $orig_mps = $restore_opts->{orig_mps};
+ my $excludes = $restore_opts->{excludes};
+ foreach my $name (keys %$excludes) {
+ push @$cmd, '--exclude', ".$orig_mps->{$name}->{mp}" if defined($orig_mps->{$name}->{mp});
+ }
if (defined($bwlimit)) {
$cmd = [ ['cstream', '-t', $bwlimit*1024], $cmd ];
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread* [pve-devel] [RFC v2 pve-container pve-manager 3/3] ui: lxc restore: add selective mountpoint restore
2023-10-23 11:18 [pve-devel] [RFC v2 pve-container pve-manager 0/3] add partial restore Christian Ebner
2023-10-23 11:18 ` [pve-devel] [RFC v2 pve-container pve-manager 1/3] backup: do not delete not backed-up mps on restore Christian Ebner
2023-10-23 11:18 ` [pve-devel] [RFC v2 pve-container pve-manager 2/3] api: allow to exclude mountpoins from restore Christian Ebner
@ 2023-10-23 11:18 ` Christian Ebner
2023-10-23 12:47 ` [pve-devel] [RFC v2 pve-container pve-manager 0/3] add partial restore Fiona Ebner
3 siblings, 0 replies; 6+ messages in thread
From: Christian Ebner @ 2023-10-23 11:18 UTC (permalink / raw)
To: pve-devel
Adds a grid to the lxc backup restore window allowing the user to select
mountpoints which should be included/excluded from a restore.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since v1:
not present in v1
www/manager6/Makefile | 1 +
www/manager6/grid/BackupRestoreTargets.js | 37 +++++++++++++++++++++++
www/manager6/window/Restore.js | 34 +++++++++++++++++++++
3 files changed, 72 insertions(+)
create mode 100644 www/manager6/grid/BackupRestoreTargets.js
diff --git a/www/manager6/Makefile b/www/manager6/Makefile
index 17e0ad05..c096a8be 100644
--- a/www/manager6/Makefile
+++ b/www/manager6/Makefile
@@ -88,6 +88,7 @@ JSSRC= \
form/TagEdit.js \
form/MultiFileButton.js \
grid/BackupView.js \
+ grid/BackupRestoreTargets.js \
grid/FirewallAliases.js \
grid/FirewallOptions.js \
grid/FirewallRules.js \
diff --git a/www/manager6/grid/BackupRestoreTargets.js b/www/manager6/grid/BackupRestoreTargets.js
new file mode 100644
index 00000000..b811989e
--- /dev/null
+++ b/www/manager6/grid/BackupRestoreTargets.js
@@ -0,0 +1,37 @@
+Ext.define('PVE.BackupRestoreTargets', {
+ extend: 'Ext.grid.Panel',
+ alias: 'widget.pveBackupRestoreTargets',
+ store: {
+ fields: [
+ {
+ name: 'restore',
+ type: 'boolean',
+ },
+ {
+ name: 'target',
+ type: 'string',
+ },
+ ],
+ },
+
+ getMountpoints: function() {
+ return this.store.getData();
+ },
+
+ setMountpoints: function(mountpoints) {
+ this.store.loadData(mountpoints);
+ },
+
+ columns: [
+ {
+ text: gettext('Mountpoint'),
+ dataIndex: 'mountpoint',
+ flex: 1,
+ },
+ {
+ text: gettext('Restore'),
+ dataIndex: 'restore',
+ xtype: 'checkcolumn',
+ },
+ ],
+});
diff --git a/www/manager6/window/Restore.js b/www/manager6/window/Restore.js
index 36aecc39..85e382fb 100644
--- a/www/manager6/window/Restore.js
+++ b/www/manager6/window/Restore.js
@@ -67,6 +67,13 @@ Ext.define('PVE.window.Restore', {
params.unprivileged = values.unprivileged;
}
confirmMsg = Proxmox.Utils.format_task_description('vzrestore', params.vmid);
+ let viewModel = view.getViewModel();
+ let excludes = viewModel.get('mountpoints')
+ .filter(mp => mp.restore === false)
+ .map(mp => mp.mountpoint);
+ if (excludes.length > 0) {
+ params['exclude-mps'] = excludes.join(',');
+ }
} else if (view.vmtype === 'qemu') {
params.archive = view.volid;
confirmMsg = Proxmox.Utils.format_task_description('qmrestore', params.vmid);
@@ -111,6 +118,7 @@ Ext.define('PVE.window.Restore', {
afterRender: function() {
let view = this.getView();
+ let viewModel = view.getViewModel();
Proxmox.Utils.API2Request({
url: `/nodes/${view.nodename}/vzdump/extractconfig`,
@@ -123,6 +131,7 @@ Ext.define('PVE.window.Restore', {
success: function(response, options) {
let allStoragesAvailable = true;
+ let mountpoints = [];
response.result.data.split('\n').forEach(line => {
let [_, key, value] = line.match(/^([^:]+):\s*(\S+)\s*$/) ?? [];
@@ -139,8 +148,14 @@ Ext.define('PVE.window.Restore', {
view.lookupReference('nameField').setEmptyText(value);
} else if (key === 'memory' || key === 'cores' || key === 'sockets') {
view.lookupReference(`${key}Field`).setEmptyText(value);
+ } else if (key.match(/mp\d+/) && value.includes('backup=1')) {
+ mountpoints.push({
+ 'mountpoint': key,
+ 'restore': true,
+ });
}
});
+ viewModel.set('mountpoints', mountpoints);
if (!allStoragesAvailable) {
let storagesel = view.down('pveStorageSelector[name=storage]');
@@ -152,6 +167,12 @@ Ext.define('PVE.window.Restore', {
},
},
+ viewModel: {
+ config: {
+ mountpoints: [],
+ },
+ },
+
initComponent: function() {
let me = this;
@@ -354,6 +375,19 @@ Ext.define('PVE.window.Restore', {
],
});
+ if (me.vmtype === 'lxc') {
+ items.push({
+ title: `${gettext('Restore Mountpoints')}:`,
+ name: 'mountpointRestoreTargets',
+ xtype: 'pveBackupRestoreTargets',
+ hidden: true, // Hide until config loaded
+ bind: {
+ mountpoints: '{mountpoints}',
+ hidden: '{!mountpoints.length}',
+ },
+ });
+ }
+
let title = gettext('Restore') + ": " + (me.vmtype === 'lxc' ? 'CT' : 'VM');
if (me.vmid) {
title = `${gettext('Overwrite')} ${title} ${me.vmid}`;
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread