From: Stoiko Ivanov <s.ivanov@proxmox.com>
To: pmg-devel@lists.proxmox.com
Subject: [pmg-devel] [PATCH pmg-api v3 4/7] Add API2 module for per-node backups to PBS
Date: Mon, 16 Nov 2020 12:01:12 +0100 [thread overview]
Message-ID: <20201116110118.7483-6-s.ivanov@proxmox.com> (raw)
In-Reply-To: <20201116110118.7483-1-s.ivanov@proxmox.com>
The module adds API2 methods for:
* creating/restoring/listing/forgetting backups on a configured PBS remote
Signed-off-by: Stoiko Ivanov <s.ivanov@proxmox.com>
---
src/Makefile | 1 +
src/PMG/API2/Nodes.pm | 7 +
src/PMG/API2/PBS/Job.pm | 371 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 379 insertions(+)
create mode 100644 src/PMG/API2/PBS/Job.pm
diff --git a/src/Makefile b/src/Makefile
index 5add6af..fb42f21 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -137,6 +137,7 @@ LIBSOURCES = \
PMG/API2/Statistics.pm \
PMG/API2/MailTracker.pm \
PMG/API2/Backup.pm \
+ PMG/API2/PBS/Job.pm \
PMG/API2/PBS/Remote.pm \
PMG/API2/Nodes.pm \
PMG/API2/Postfix.pm \
diff --git a/src/PMG/API2/Nodes.pm b/src/PMG/API2/Nodes.pm
index 96aa146..259f8f3 100644
--- a/src/PMG/API2/Nodes.pm
+++ b/src/PMG/API2/Nodes.pm
@@ -26,6 +26,7 @@ use PMG::API2::SpamAssassin;
use PMG::API2::Postfix;
use PMG::API2::MailTracker;
use PMG::API2::Backup;
+use PMG::API2::PBS::Job;
use base qw(PVE::RESTHandler);
@@ -79,6 +80,11 @@ __PACKAGE__->register_method ({
path => 'backup',
});
+__PACKAGE__->register_method ({
+ subclass => "PMG::API2::PBS::Job",
+ path => 'pbs',
+});
+
__PACKAGE__->register_method ({
name => 'index',
path => '',
@@ -105,6 +111,7 @@ __PACKAGE__->register_method ({
my $result = [
{ name => 'apt' },
{ name => 'backup' },
+ { name => 'pbs' },
{ name => 'clamav' },
{ name => 'spamassassin' },
{ name => 'postfix' },
diff --git a/src/PMG/API2/PBS/Job.pm b/src/PMG/API2/PBS/Job.pm
new file mode 100644
index 0000000..dee1754
--- /dev/null
+++ b/src/PMG/API2/PBS/Job.pm
@@ -0,0 +1,371 @@
+package PMG::API2::PBS::Job;
+
+use strict;
+use warnings;
+
+use POSIX qw(strftime);
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RESTHandler;
+use PVE::SafeSyslog;
+use PVE::Tools qw(extract_param);
+use PVE::PBSClient;
+
+use PMG::RESTEnvironment;
+use PMG::Backup;
+use PMG::PBSConfig;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+ name => 'list',
+ path => '',
+ method => 'GET',
+ description => "List all configured Proxmox Backup Server jobs.",
+ permissions => { check => [ 'admin', 'audit' ] },
+ proxyto => 'node',
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ },
+ },
+ returns => {
+ type => "array",
+ items => PMG::PBSConfig->createSchema(1),
+ links => [ { rel => 'child', href => "{remote}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $res = [];
+
+ my $conf = PMG::PBSConfig->new();
+ if (defined($conf)) {
+ foreach my $remote (keys %{$conf->{ids}}) {
+ my $d = $conf->{ids}->{$remote};
+ my $entry = {
+ remote => $remote,
+ server => $d->{server},
+ datastore => $d->{datastore},
+ };
+ push @$res, $entry;
+ }
+ }
+
+ return $res;
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'remote_index',
+ path => '{remote}',
+ method => 'GET',
+ description => "Backup Job index.",
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => "object",
+ properties => { section => { type => 'string'} },
+ },
+ links => [ { rel => 'child', href => "{section}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $result = [
+ { section => 'snapshots' },
+ { section => 'backup' },
+ { section => 'restore' },
+ { section => 'timer' },
+ ];
+ return $result;
+}});
+
+__PACKAGE__->register_method ({
+ name => 'get_snapshots',
+ path => '{remote}/snapshots',
+ method => 'GET',
+ description => "Get snapshots stored on remote.",
+ proxyto => 'node',
+ protected => 1,
+ permissions => { check => [ 'admin', 'audit' ] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => "object",
+ properties => {
+ time => { type => 'string'},
+ ctime => { type => 'string'},
+ size => { type => 'integer'},
+ },
+ },
+ links => [ { rel => 'child', href => "{time}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $remote = $param->{remote};
+ my $node = $param->{node};
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+
+ return [] if $remote_config->{disable};
+
+ my $snap_param = {
+ group => "host/$node",
+ };
+
+ my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir});
+ my $snapshots = $pbs->get_snapshots($snap_param);
+ my $res = [];
+ foreach my $item (@$snapshots) {
+ my $btype = $item->{"backup-type"};
+ my $bid = $item->{"backup-id"};
+ my $epoch = $item->{"backup-time"};
+ my $size = $item->{size} // 1;
+
+ my @pxar = grep { $_->{filename} eq 'pmgbackup.pxar.didx' } @{$item->{files}};
+ die "unexpected number of pmgbackup archives in snapshot\n" if (scalar(@pxar) != 1);
+
+
+ next if !($btype eq 'host');
+ next if !($bid eq $node);
+
+ my $time = strftime("%FT%TZ", gmtime($epoch));
+
+ my $info = {
+ time => $time,
+ ctime => $epoch,
+ size => $size,
+ };
+
+ push @$res, $info;
+ }
+
+ return $res;
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'forget_snapshot',
+ path => '{remote}/snapshots/{time}',
+ method => 'DELETE',
+ description => "Forget a snapshot",
+ proxyto => 'node',
+ protected => 1,
+ permissions => { check => [ 'admin', 'audit' ] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ time => {
+ description => "Backup time in RFC 3399 format",
+ type => 'string',
+ },
+ },
+ },
+ returns => {type => 'null' },
+ code => sub {
+ my ($param) = @_;
+
+ my $remote = $param->{remote};
+ my $node = $param->{node};
+ my $time = $param->{time};
+
+ my $snapshot = "host/$node/$time";
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+ die "PBS remote '$remote' is disabled\n" if $remote_config->{disable};
+
+ my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir});
+
+ eval {
+ $pbs->forget_snapshot($snapshot);
+ };
+ die "Forgetting backup failed: $@" if $@;
+
+ return;
+
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'run_backup',
+ path => '{remote}/backup',
+ method => 'POST',
+ description => "run backup and prune the backupgroup afterwards.",
+ proxyto => 'node',
+ protected => 1,
+ permissions => { check => [ 'admin', 'audit' ] },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ },
+ },
+ returns => { type => "string" },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PMG::RESTEnvironment->get();
+ my $authuser = $rpcenv->get_user();
+
+ my $remote = $param->{remote};
+ my $node = $param->{node};
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+ die "PBS remote '$remote' is disabled\n" if $remote_config->{disable};
+
+ my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir});
+ my $backup_dir = "/var/lib/pmg/backup/current";
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "starting update of current backup state\n";
+
+ -d $backup_dir || mkdir $backup_dir;
+ PMG::Backup::pmg_backup($backup_dir, $param->{statistic});
+ my $pbs_opts = {
+ type => 'host',
+ id => $node,
+ pxarname => 'pmgbackup',
+ root => $backup_dir,
+ };
+
+ $pbs->backup_tree($pbs_opts);
+
+ print "backup finished\n";
+
+ my $group = "host/$node";
+ print "starting prune of $group\n";
+ my $prune_opts = $conf->prune_options($remote);
+ my $res = $pbs->prune_group(undef, $prune_opts, $group);
+
+ foreach my $pruned (@$res){
+ my $time = strftime("%FT%TZ", gmtime($pruned->{'backup-time'}));
+ my $snap = $pruned->{'backup-type'} . '/' . $pruned->{'backup-id'} . '/' . $time;
+ print "pruned snapshot: $snap\n";
+ }
+
+ print "prune finished\n";
+
+ return;
+ };
+
+ return $rpcenv->fork_worker('pbs_backup', undef, $authuser, $worker);
+
+ }});
+
+__PACKAGE__->register_method ({
+ name => 'restore',
+ path => '{remote}/restore',
+ method => 'POST',
+ description => "Restore the system configuration.",
+ permissions => { check => [ 'admin' ] },
+ proxyto => 'node',
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ PMG::Backup::get_restore_options(),
+ remote => {
+ description => "Proxmox Backup Server ID.",
+ type => 'string', format => 'pve-configid',
+ },
+ 'backup-time' => {description=> "backup-time to restore",
+ optional => 1, type => 'string'
+ },
+ 'backup-id' => {description => "backup-id (hostname) of backup snapshot",
+ optional => 1, type => 'string'
+ },
+ },
+ },
+ returns => { type => "string" },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PMG::RESTEnvironment->get();
+ my $authuser = $rpcenv->get_user();
+
+ my $remote = $param->{remote};
+ my $backup_id = $param->{'backup-id'} // $param->{node};
+ my $snapshot = "host/$backup_id";
+ $snapshot .= "/$param->{'backup-time'}" if defined($param->{'backup-time'});
+
+ my $conf = PMG::PBSConfig->new();
+
+ my $remote_config = $conf->{ids}->{$remote};
+ die "PBS remote '$remote' does not exist\n" if !$remote_config;
+ die "PBS remote '$remote' is disabled\n" if $remote_config->{disable};
+
+ my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir});
+
+ my $time = time;
+ my $dirname = "/tmp/proxrestore_$$.$time";
+
+ $param->{database} //= 1;
+
+ die "nothing selected - please select what you want to restore (config or database?)\n"
+ if !($param->{database} || $param->{config});
+
+ my $pbs_opts = {
+ pxarname => 'pmgbackup',
+ target => $dirname,
+ snapshot => $snapshot,
+ };
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "starting restore of $snapshot from $remote\n";
+
+ $pbs->restore_pxar($pbs_opts);
+ print "starting restore of PMG config\n";
+ PMG::Backup::pmg_restore($dirname, $param->{database},
+ $param->{config}, $param->{statistic});
+ print "restore finished\n";
+
+ return;
+ };
+
+ return $rpcenv->fork_worker('pbs_restore', undef, $authuser, $worker);
+ }});
+
+1;
--
2.20.1
next prev parent reply other threads:[~2020-11-16 11:02 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-11-16 11:01 [pmg-devel] [PATCH pve-common/pmg-api/pmg-gui v3] add initial PBS integration Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pve-common v3 1/1] add PBSClient module Stoiko Ivanov
2020-11-17 8:49 ` [pmg-devel] applied: " Thomas Lamprecht
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-api v3 1/7] debian: drop duplicate ', ' in dependencies Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-api v3 2/7] add initial SectionConfig for PBS Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-api v3 3/7] Add API2 module for PBS configuration Stoiko Ivanov
2020-11-16 11:01 ` Stoiko Ivanov [this message]
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-api v3 5/7] pbs-integration: add CLI calls to pmgbackup Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-api v3 6/7] add scheduled backup to PBS remotes Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-api v3 7/7] add /etc/pmg/pbs to cluster-sync Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-gui v3 1/3] Make Backup/Restore panel a menuentry Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-gui v3 2/3] refactor RestoreWindow for PBS Stoiko Ivanov
2020-11-16 11:01 ` [pmg-devel] [PATCH pmg-gui v3 3/3] add PBSConfig tab to Backup menu Stoiko Ivanov
2020-11-17 17:22 ` [pmg-devel] applied-series: [PATCH pve-common/pmg-api/pmg-gui v3] add initial PBS integration Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201116110118.7483-6-s.ivanov@proxmox.com \
--to=s.ivanov@proxmox.com \
--cc=pmg-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox