From: Stoiko Ivanov <s.ivanov@proxmox.com>
To: pmg-devel@pve.proxmox.com
Subject: [pmg-devel] [PATCH pmg-api 08/11] Add API2 module for per-node backups to PBS
Date: Wed, 28 Oct 2020 19:54:26 +0100 [thread overview]
Message-ID: <20201028185432.23067-11-s.ivanov@proxmox.com> (raw)
In-Reply-To: <20201028185432.23067-1-s.ivanov@proxmox.com>
The module adds API2 methods for:
* creating/restoring/listing/forgetting backups on a configured PBS remote
Signed-off-by: Stoiko Ivanov <s.ivanov@proxmox.com>
---
src/Makefile | 1 +
src/PMG/API2/Nodes.pm | 7 +
src/PMG/API2/PBS/Job.pm | 413 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 421 insertions(+)
create mode 100644 src/PMG/API2/PBS/Job.pm
diff --git a/src/Makefile b/src/Makefile
index 5add6af..fb42f21 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -137,6 +137,7 @@ LIBSOURCES = \
PMG/API2/Statistics.pm \
PMG/API2/MailTracker.pm \
PMG/API2/Backup.pm \
+ PMG/API2/PBS/Job.pm \
PMG/API2/PBS/Remote.pm \
PMG/API2/Nodes.pm \
PMG/API2/Postfix.pm \
diff --git a/src/PMG/API2/Nodes.pm b/src/PMG/API2/Nodes.pm
index 96aa146..259f8f3 100644
--- a/src/PMG/API2/Nodes.pm
+++ b/src/PMG/API2/Nodes.pm
@@ -26,6 +26,7 @@ use PMG::API2::SpamAssassin;
use PMG::API2::Postfix;
use PMG::API2::MailTracker;
use PMG::API2::Backup;
+use PMG::API2::PBS::Job;
use base qw(PVE::RESTHandler);
@@ -79,6 +80,11 @@ __PACKAGE__->register_method ({
path => 'backup',
});
+__PACKAGE__->register_method ({
+ subclass => "PMG::API2::PBS::Job",
+ path => 'pbs',
+});
+
__PACKAGE__->register_method ({
name => 'index',
path => '',
@@ -105,6 +111,7 @@ __PACKAGE__->register_method ({
my $result = [
{ name => 'apt' },
{ name => 'backup' },
+ { name => 'pbs' },
{ name => 'clamav' },
{ name => 'spamassassin' },
{ name => 'postfix' },
diff --git a/src/PMG/API2/PBS/Job.pm b/src/PMG/API2/PBS/Job.pm
new file mode 100644
index 0000000..80a1f29
--- /dev/null
+++ b/src/PMG/API2/PBS/Job.pm
@@ -0,0 +1,413 @@
+package PMG::API2::PBS::Job;
+
+use strict;
+use warnings;
+
+use POSIX qw(strftime);
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RESTHandler;
+use PVE::SafeSyslog;
+use PVE::Tools qw(extract_param);
+use PVE::PBSTools;
+
+use PMG::RESTEnvironment;
+use PMG::Backup;
+use PMG::PBSConfig;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+ name => 'list',
+ path => '',
+ method => 'GET',
+ description => "List all configured Proxmox Backup Server jobs.",
+ permissions => { check => [ 'admin', 'audit' ] },
+ proxyto => 'node',
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ },
+ },
+ returns => {
+ type => "array",
+ items => PMG::PBSConfig->createSchema(1),
+ links => [ { rel => 'child', href => "{remote}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $res = [];
+
+ my $conf = PMG::PBSConfig->new();
+ if (defined($conf)) {
+ foreach my $remote (keys %{$conf->{ids}}) {
+ my $d = $conf->{ids}->{$remote};
+ my $entry = {
+ remote => $remote,
+ server => $d->{server},
+ datastore => $d->{datastore},
+ };
+ push @$res, $entry;
+ }
+ }
+
+ return $res;
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'remote_index',
+ path => '{remote}',
+ method => 'GET',
+ description => "Backup Job index.",
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => "object",
+ properties => { section => { type => 'string'} },
+ },
+ links => [ { rel => 'child', href => "{section}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $result = [
+ { section => 'snapshots' },
+ { section => 'backup' },
+ { section => 'restore' },
+ { section => 'timer' },
+ { section => 'prune' },
+ ];
+ return $result;
+}});
+
+__PACKAGE__->register_method ({
+ name => 'get_snapshots',
+ path => '{remote}/snapshots',
+ method => 'GET',
+ description => "Get snapshots stored on remote.",
+ proxyto => 'node',
+ protected => 1,
+ permissions => { check => [ 'admin', 'audit' ] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => "object",
+ properties => {
+ time => { type => 'string'},
+ ctime => { type => 'string'},
+ size => { type => 'integer'},
+ encrypted => { type => 'boolean'},
+ },
+ },
+ links => [ { rel => 'child', href => "{time}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $remote = $param->{remote};
+ my $node = $param->{node};
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+
+ return [] if $remote_config->{disable};
+
+ my $snap_param = {
+ group => "host/$node",
+ };
+ my $snapshots = PVE::PBSTools::get_snapshots($remote_config, $remote, $snap_param);
+ my $res = [];
+ foreach my $item (@$snapshots) {
+ my $btype = $item->{"backup-type"};
+ my $bid = $item->{"backup-id"};
+ my $epoch = $item->{"backup-time"};
+ my $size = $item->{size} // 1;
+
+ my @pxar = grep { $_->{filename} eq 'pmgbackup.pxar.didx' } @{$item->{files}};
+ die "unexpected number of pmgbackup archives in snapshot\n" if (scalar(@pxar) != 1);
+
+ my $encrypted = ($pxar[0]->{'crypt-mode'} eq 'encrypt') ? 1 : 0;
+
+
+ next if !($btype eq 'host');
+ next if !($bid eq $node);
+
+ my $time = strftime("%FT%TZ", gmtime($epoch));
+
+ my $info = {
+ time => $time,
+ ctime => $epoch,
+ size => $size,
+ encrypted => $encrypted,
+ };
+
+ push @$res, $info;
+ }
+
+ return $res;
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'forget_snapshot',
+ path => '{remote}/snapshots/{time}',
+ method => 'DELETE',
+ description => "Forget a snapshot",
+ proxyto => 'node',
+ protected => 1,
+ permissions => { check => [ 'admin', 'audit' ] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ time => {
+ description => "Backup time in RFC 3399 format",
+ type => 'string',
+ },
+ },
+ },
+ returns => {type => 'null' },
+ code => sub {
+ my ($param) = @_;
+
+ my $remote = $param->{remote};
+ my $node = $param->{node};
+ my $time = $param->{time};
+
+ my $snapshot = "host/$node/$time";
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+ die "PBS remote '$remote' is disabled\n" if $remote_config->{disable};
+
+ eval {
+ PVE::PBSTools::forget_snapshot($remote_config, $remote, $snapshot);
+ };
+ die "Forgetting backup failed: $@" if $@;
+
+ return;
+
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'run_backup',
+ path => '{remote}/backup',
+ method => 'POST',
+ description => "run backup",
+ proxyto => 'node',
+ protected => 1,
+ permissions => { check => [ 'admin', 'audit' ] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ },
+ },
+ returns => { type => "string" },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PMG::RESTEnvironment->get();
+ my $authuser = $rpcenv->get_user();
+
+ my $remote = $param->{remote};
+ my $node = $param->{node};
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+ die "PBS remote '$remote' is disabled\n" if $remote_config->{disable};
+
+ my $backup_dir = "/var/lib/pmg/backup/current";
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "starting update of current backup state\n";
+
+ PMG::Backup::pmg_backup($backup_dir, $param->{statistic});
+ my $pbs_opts = {
+ type => 'host',
+ id => $node,
+ pxarname => 'pmgbackup',
+ root => $backup_dir,
+ };
+
+ PVE::PBSTools::backup_tree($remote_config, $remote, $pbs_opts);
+
+ print "backup finished\n";
+
+ return;
+ };
+
+ return $rpcenv->fork_worker('pbs_backup', undef, $authuser, $worker);
+
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'restore',
+ path => '{remote}/restore',
+ method => 'POST',
+ description => "Restore the system configuration.",
+ permissions => { check => [ 'admin' ] },
+ proxyto => 'node',
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ PMG::Backup::get_restore_options(),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ 'backup-time' => {description=> "backup-time to restore",
+ optional => 1, type => 'string'
+ },
+ 'backup-id' => {description => "backup-id (hostname) of backup snapshot",
+ optional => 1, type => 'string'
+ },
+ },
+ },
+ returns => { type => "string" },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PMG::RESTEnvironment->get();
+ my $authuser = $rpcenv->get_user();
+
+ my $remote = $param->{remote};
+ my $backup_id = $param->{'backup-id'} // $param->{node};
+ my $snapshot = "host/$backup_id";
+ $snapshot .= "/$param->{'backup-time'}" if defined($param->{'backup-time'});
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+ die "PBS remote '$remote' is disabled\n" if $remote_config->{disable};
+
+ my $time = time;
+ my $dirname = "/tmp/proxrestore_$$.$time";
+
+ $param->{database} //= 1;
+
+ die "nothing selected - please select what you want to restore (config or database?)\n"
+ if !($param->{database} || $param->{config});
+
+ my $pbs_opts = {
+ pxarname => 'pmgbackup',
+ target => $dirname,
+ snapshot => $snapshot,
+ };
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "starting restore of $snapshot from $remote\n";
+
+ PVE::PBSTools::restore_pxar($remote_config, $remote, $pbs_opts);
+ print "starting restore of PMG config\n";
+ PMG::Backup::pmg_restore($dirname, $param->{database},
+ $param->{config}, $param->{statistic});
+ print "restore finished\n";
+
+ return;
+ };
+
+ return $rpcenv->fork_worker('pbs_restore', undef, $authuser, $worker);
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'run_prune',
+ path => '{remote}/prune',
+ method => 'POST',
+ description => "Prune the remote based on the configured options",
+ proxyto => 'node',
+ protected => 1,
+ permissions => { check => [ 'admin', 'audit' ] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ },
+ },
+ returns => { type => "string" },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PMG::RESTEnvironment->get();
+ my $authuser = $rpcenv->get_user();
+
+ my $remote = $param->{remote};
+ my $node = $param->{node};
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+ die "PBS remote '$remote' is disabled\n" if $remote_config->{disable};
+
+ my $group = "host/$node";
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "starting prune of current backup state\n";
+
+ my $prune_opts = $conf->prune_options($remote);
+ my $res = PVE::PBSTools::prune_group($remote_config, $remote, $prune_opts, $group);
+
+ foreach my $pruned (@$res){
+ my $time = strftime("%FT%TZ", gmtime($pruned->{'backup-time'}));
+ my $snap = $pruned->{'backup-type'} . '/' . $pruned->{'backup-id'} . '/' . $time;
+ print "pruned snapshot: $snap\n";
+ }
+
+ print "prune finished\n";
+
+ return;
+ };
+
+ return $rpcenv->fork_worker('pbs_backup', undef, $authuser, $worker);
+
+ }});
+1;
--
2.20.1
next prev parent reply other threads:[~2020-10-28 18:54 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-10-28 18:54 [pmg-devel] [PATCH pve-common/api/gui] add initial PBS integration Stoiko Ivanov
2020-10-28 18:54 ` [pmg-devel] [PATCH pve-common 1/2] Systemd: add helpers for parsing unit files Stoiko Ivanov
2020-10-30 8:26 ` [pmg-devel] applied: " Dietmar Maurer
2020-11-10 8:34 ` [pmg-devel] " Thomas Lamprecht
2020-10-28 18:54 ` [pmg-devel] [PATCH pve-common 2/2] add helper module for handling PBS Integration Stoiko Ivanov
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 01/11] drop left-over commented out code Stoiko Ivanov
2020-10-30 5:58 ` [pmg-devel] applied: " Dietmar Maurer
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 02/11] Backup: split backup creation and creating tar Stoiko Ivanov
2020-10-30 6:20 ` [pmg-devel] applied: " Dietmar Maurer
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 03/11] Restore: optionally restore from directory Stoiko Ivanov
2020-10-30 6:26 ` [pmg-devel] applied: " Dietmar Maurer
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 04/11] Backup: push restore options to PMG::Backup Stoiko Ivanov
2020-10-30 6:32 ` [pmg-devel] applied: " Dietmar Maurer
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 05/11] debian: add dependency on proxmox-backup-client Stoiko Ivanov
2020-10-30 6:33 ` [pmg-devel] applied: " Dietmar Maurer
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 06/11] add initial SectionConfig for pbs Stoiko Ivanov
2020-10-30 6:38 ` Dietmar Maurer
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 07/11] Add API2 module for PBS configuration Stoiko Ivanov
2020-10-28 18:54 ` Stoiko Ivanov [this message]
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 09/11] pbs-integration: add CLI calls to pmgbackup Stoiko Ivanov
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 10/11] add scheduled backup to PBS remotes Stoiko Ivanov
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-api 11/11] add daily timer for pruning configured remotes Stoiko Ivanov
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-gui 1/3] Make Backup/Restore panel a menuentry Stoiko Ivanov
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-gui 2/3] refactor RestoreWindow for PBS Stoiko Ivanov
2020-10-28 18:54 ` [pmg-devel] [PATCH pmg-gui 3/3] add PBSConfig tab to Backup menu Stoiko Ivanov
2020-10-30 5:47 ` [pmg-devel] [PATCH pve-common/api/gui] add initial PBS integration Dietmar Maurer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201028185432.23067-11-s.ivanov@proxmox.com \
--to=s.ivanov@proxmox.com \
--cc=pmg-devel@pve.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox