From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 0D41969F3D for ; Mon, 16 Nov 2020 12:02:08 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 026E52DB4C for ; Mon, 16 Nov 2020 12:01:38 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id E97462DAEC for ; Mon, 16 Nov 2020 12:01:32 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id B048E436BF for ; Mon, 16 Nov 2020 12:01:32 +0100 (CET) From: Stoiko Ivanov To: pmg-devel@lists.proxmox.com Date: Mon, 16 Nov 2020 12:01:12 +0100 Message-Id: <20201116110118.7483-6-s.ivanov@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201116110118.7483-1-s.ivanov@proxmox.com> References: <20201116110118.7483-1-s.ivanov@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.284 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [nodes.pm, job.pm, remote.pm, backup.pm, postfix.pm, statistics.pm, mailtracker.pm] URIBL_SBL 0.644 Contains an URL's NS IP listed in the Spamhaus SBL blocklist [backup.pm] URIBL_SBL_A 0.1 Contains URL's A record listed in the Spamhaus SBL blocklist [backup.pm] Subject: [pmg-devel] [PATCH pmg-api v3 4/7] Add API2 module for per-node backups to PBS X-BeenThere: pmg-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Mail Gateway development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 16 Nov 2020 11:02:08 -0000 The module adds API2 methods for: * creating/restoring/listing/forgetting backups on a configured PBS remote Signed-off-by: Stoiko Ivanov --- src/Makefile | 1 + src/PMG/API2/Nodes.pm | 7 + src/PMG/API2/PBS/Job.pm | 371 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 379 insertions(+) create mode 100644 src/PMG/API2/PBS/Job.pm diff --git a/src/Makefile b/src/Makefile index 5add6af..fb42f21 100644 --- a/src/Makefile +++ b/src/Makefile @@ -137,6 +137,7 @@ LIBSOURCES = \ PMG/API2/Statistics.pm \ PMG/API2/MailTracker.pm \ PMG/API2/Backup.pm \ + PMG/API2/PBS/Job.pm \ PMG/API2/PBS/Remote.pm \ PMG/API2/Nodes.pm \ PMG/API2/Postfix.pm \ diff --git a/src/PMG/API2/Nodes.pm b/src/PMG/API2/Nodes.pm index 96aa146..259f8f3 100644 --- a/src/PMG/API2/Nodes.pm +++ b/src/PMG/API2/Nodes.pm @@ -26,6 +26,7 @@ use PMG::API2::SpamAssassin; use PMG::API2::Postfix; use PMG::API2::MailTracker; use PMG::API2::Backup; +use PMG::API2::PBS::Job; use base qw(PVE::RESTHandler); @@ -79,6 +80,11 @@ __PACKAGE__->register_method ({ path => 'backup', }); +__PACKAGE__->register_method ({ + subclass => "PMG::API2::PBS::Job", + path => 'pbs', +}); + __PACKAGE__->register_method ({ name => 'index', path => '', @@ -105,6 +111,7 @@ __PACKAGE__->register_method ({ my $result = [ { name => 'apt' }, { name => 'backup' }, + { name => 'pbs' }, { name => 'clamav' }, { name => 'spamassassin' }, { name => 'postfix' }, diff --git a/src/PMG/API2/PBS/Job.pm b/src/PMG/API2/PBS/Job.pm new file mode 100644 index 0000000..dee1754 --- /dev/null +++ b/src/PMG/API2/PBS/Job.pm @@ -0,0 +1,371 @@ +package PMG::API2::PBS::Job; + +use strict; +use warnings; + +use POSIX qw(strftime); + +use PVE::JSONSchema qw(get_standard_option); +use PVE::RESTHandler; +use PVE::SafeSyslog; +use PVE::Tools qw(extract_param); +use PVE::PBSClient; + +use PMG::RESTEnvironment; +use PMG::Backup; +use PMG::PBSConfig; + +use base qw(PVE::RESTHandler); + +__PACKAGE__->register_method ({ + name => 'list', + path => '', + method => 'GET', + description => "List all configured Proxmox Backup Server jobs.", + permissions => { check => [ 'admin', 'audit' ] }, + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => "array", + items => PMG::PBSConfig->createSchema(1), + links => [ { rel => 'child', href => "{remote}" } ], + }, + code => sub { + my ($param) = @_; + + my $res = []; + + my $conf = PMG::PBSConfig->new(); + if (defined($conf)) { + foreach my $remote (keys %{$conf->{ids}}) { + my $d = $conf->{ids}->{$remote}; + my $entry = { + remote => $remote, + server => $d->{server}, + datastore => $d->{datastore}, + }; + push @$res, $entry; + } + } + + return $res; + }}); + +__PACKAGE__->register_method ({ + name => 'remote_index', + path => '{remote}', + method => 'GET', + description => "Backup Job index.", + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { section => { type => 'string'} }, + }, + links => [ { rel => 'child', href => "{section}" } ], + }, + code => sub { + my ($param) = @_; + + my $result = [ + { section => 'snapshots' }, + { section => 'backup' }, + { section => 'restore' }, + { section => 'timer' }, + ]; + return $result; +}}); + +__PACKAGE__->register_method ({ + name => 'get_snapshots', + path => '{remote}/snapshots', + method => 'GET', + description => "Get snapshots stored on remote.", + proxyto => 'node', + protected => 1, + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { + time => { type => 'string'}, + ctime => { type => 'string'}, + size => { type => 'integer'}, + }, + }, + links => [ { rel => 'child', href => "{time}" } ], + }, + code => sub { + my ($param) = @_; + + my $remote = $param->{remote}; + my $node = $param->{node}; + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + + return [] if $remote_config->{disable}; + + my $snap_param = { + group => "host/$node", + }; + + my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir}); + my $snapshots = $pbs->get_snapshots($snap_param); + my $res = []; + foreach my $item (@$snapshots) { + my $btype = $item->{"backup-type"}; + my $bid = $item->{"backup-id"}; + my $epoch = $item->{"backup-time"}; + my $size = $item->{size} // 1; + + my @pxar = grep { $_->{filename} eq 'pmgbackup.pxar.didx' } @{$item->{files}}; + die "unexpected number of pmgbackup archives in snapshot\n" if (scalar(@pxar) != 1); + + + next if !($btype eq 'host'); + next if !($bid eq $node); + + my $time = strftime("%FT%TZ", gmtime($epoch)); + + my $info = { + time => $time, + ctime => $epoch, + size => $size, + }; + + push @$res, $info; + } + + return $res; + }}); + +__PACKAGE__->register_method ({ + name => 'forget_snapshot', + path => '{remote}/snapshots/{time}', + method => 'DELETE', + description => "Forget a snapshot", + proxyto => 'node', + protected => 1, + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + time => { + description => "Backup time in RFC 3399 format", + type => 'string', + }, + }, + }, + returns => {type => 'null' }, + code => sub { + my ($param) = @_; + + my $remote = $param->{remote}; + my $node = $param->{node}; + my $time = $param->{time}; + + my $snapshot = "host/$node/$time"; + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + die "PBS remote '$remote' is disabled\n" if $remote_config->{disable}; + + my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir}); + + eval { + $pbs->forget_snapshot($snapshot); + }; + die "Forgetting backup failed: $@" if $@; + + return; + + }}); + +__PACKAGE__->register_method ({ + name => 'run_backup', + path => '{remote}/backup', + method => 'POST', + description => "run backup and prune the backupgroup afterwards.", + proxyto => 'node', + protected => 1, + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { type => "string" }, + code => sub { + my ($param) = @_; + + my $rpcenv = PMG::RESTEnvironment->get(); + my $authuser = $rpcenv->get_user(); + + my $remote = $param->{remote}; + my $node = $param->{node}; + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + die "PBS remote '$remote' is disabled\n" if $remote_config->{disable}; + + my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir}); + my $backup_dir = "/var/lib/pmg/backup/current"; + + my $worker = sub { + my $upid = shift; + + print "starting update of current backup state\n"; + + -d $backup_dir || mkdir $backup_dir; + PMG::Backup::pmg_backup($backup_dir, $param->{statistic}); + my $pbs_opts = { + type => 'host', + id => $node, + pxarname => 'pmgbackup', + root => $backup_dir, + }; + + $pbs->backup_tree($pbs_opts); + + print "backup finished\n"; + + my $group = "host/$node"; + print "starting prune of $group\n"; + my $prune_opts = $conf->prune_options($remote); + my $res = $pbs->prune_group(undef, $prune_opts, $group); + + foreach my $pruned (@$res){ + my $time = strftime("%FT%TZ", gmtime($pruned->{'backup-time'})); + my $snap = $pruned->{'backup-type'} . '/' . $pruned->{'backup-id'} . '/' . $time; + print "pruned snapshot: $snap\n"; + } + + print "prune finished\n"; + + return; + }; + + return $rpcenv->fork_worker('pbs_backup', undef, $authuser, $worker); + + }}); + +__PACKAGE__->register_method ({ + name => 'restore', + path => '{remote}/restore', + method => 'POST', + description => "Restore the system configuration.", + permissions => { check => [ 'admin' ] }, + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + PMG::Backup::get_restore_options(), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + 'backup-time' => {description=> "backup-time to restore", + optional => 1, type => 'string' + }, + 'backup-id' => {description => "backup-id (hostname) of backup snapshot", + optional => 1, type => 'string' + }, + }, + }, + returns => { type => "string" }, + code => sub { + my ($param) = @_; + + my $rpcenv = PMG::RESTEnvironment->get(); + my $authuser = $rpcenv->get_user(); + + my $remote = $param->{remote}; + my $backup_id = $param->{'backup-id'} // $param->{node}; + my $snapshot = "host/$backup_id"; + $snapshot .= "/$param->{'backup-time'}" if defined($param->{'backup-time'}); + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + die "PBS remote '$remote' is disabled\n" if $remote_config->{disable}; + + my $pbs = PVE::PBSClient->new($remote_config, $remote, $conf->{secret_dir}); + + my $time = time; + my $dirname = "/tmp/proxrestore_$$.$time"; + + $param->{database} //= 1; + + die "nothing selected - please select what you want to restore (config or database?)\n" + if !($param->{database} || $param->{config}); + + my $pbs_opts = { + pxarname => 'pmgbackup', + target => $dirname, + snapshot => $snapshot, + }; + + my $worker = sub { + my $upid = shift; + + print "starting restore of $snapshot from $remote\n"; + + $pbs->restore_pxar($pbs_opts); + print "starting restore of PMG config\n"; + PMG::Backup::pmg_restore($dirname, $param->{database}, + $param->{config}, $param->{statistic}); + print "restore finished\n"; + + return; + }; + + return $rpcenv->fork_worker('pbs_restore', undef, $authuser, $worker); + }}); + +1; -- 2.20.1