From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5DA0E60DBE for ; Mon, 19 Oct 2020 21:02:28 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 5B6BF2F1C0 for ; Mon, 19 Oct 2020 21:02:28 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C00B42F171 for ; Mon, 19 Oct 2020 21:02:24 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9286D444A5 for ; Mon, 19 Oct 2020 21:02:24 +0200 (CEST) From: Stoiko Ivanov To: pmg-devel@lists.proxmox.com Date: Mon, 19 Oct 2020 21:02:08 +0200 Message-Id: <20201019190209.11495-12-s.ivanov@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201019190209.11495-1-s.ivanov@proxmox.com> References: <20201019190209.11495-1-s.ivanov@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.310 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_SBL 0.644 Contains an URL's NS IP listed in the Spamhaus SBL blocklist [backup.pm] URIBL_SBL_A 0.1 Contains URL's A record listed in the Spamhaus SBL blocklist [backup.pm] Subject: [pmg-devel] [RFC pmg-api 11/12] Add API2 module for per-node backups to PBS X-BeenThere: pmg-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Mail Gateway development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 19 Oct 2020 19:02:28 -0000 The module adds API2 methods for: * creating/restoring/listing/forgetting backups to a configured PBS remote * creating backup schedules (using systemd-timers) Signed-off-by: Stoiko Ivanov --- src/Makefile | 1 + src/PMG/API2/Nodes.pm | 7 + src/PMG/API2/PBS/Job.pm | 467 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 475 insertions(+) create mode 100644 src/PMG/API2/PBS/Job.pm diff --git a/src/Makefile b/src/Makefile index e1546a8..b978cfa 100644 --- a/src/Makefile +++ b/src/Makefile @@ -138,6 +138,7 @@ LIBSOURCES = \ PMG/API2/Statistics.pm \ PMG/API2/MailTracker.pm \ PMG/API2/Backup.pm \ + PMG/API2/PBS/Job.pm \ PMG/API2/PBS/Remote.pm \ PMG/API2/Nodes.pm \ PMG/API2/Postfix.pm \ diff --git a/src/PMG/API2/Nodes.pm b/src/PMG/API2/Nodes.pm index 96aa146..259f8f3 100644 --- a/src/PMG/API2/Nodes.pm +++ b/src/PMG/API2/Nodes.pm @@ -26,6 +26,7 @@ use PMG::API2::SpamAssassin; use PMG::API2::Postfix; use PMG::API2::MailTracker; use PMG::API2::Backup; +use PMG::API2::PBS::Job; use base qw(PVE::RESTHandler); @@ -79,6 +80,11 @@ __PACKAGE__->register_method ({ path => 'backup', }); +__PACKAGE__->register_method ({ + subclass => "PMG::API2::PBS::Job", + path => 'pbs', +}); + __PACKAGE__->register_method ({ name => 'index', path => '', @@ -105,6 +111,7 @@ __PACKAGE__->register_method ({ my $result = [ { name => 'apt' }, { name => 'backup' }, + { name => 'pbs' }, { name => 'clamav' }, { name => 'spamassassin' }, { name => 'postfix' }, diff --git a/src/PMG/API2/PBS/Job.pm b/src/PMG/API2/PBS/Job.pm new file mode 100644 index 0000000..8fa3a19 --- /dev/null +++ b/src/PMG/API2/PBS/Job.pm @@ -0,0 +1,467 @@ +package PMG::API2::PBS::Job; + +use strict; +use warnings; + +use POSIX qw(strftime); + +use PVE::JSONSchema qw(get_standard_option); +use PVE::RESTHandler; +use PVE::SafeSyslog; +use PVE::Tools qw(extract_param); + +use PMG::RESTEnvironment; +use PMG::Backup; +use PMG::PBSTools; +use PMG::PBSConfig; + +use base qw(PVE::RESTHandler); + +__PACKAGE__->register_method ({ + name => 'list', + path => '', + method => 'GET', + description => "List all configured Proxmox Backup Server jobs.", + permissions => { check => [ 'admin', 'audit' ] }, + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => "array", + items => PMG::PBSConfig->createSchema(1), + links => [ { rel => 'child', href => "{remote}" } ], + }, + code => sub { + my ($param) = @_; + + my $res = []; + + my $conf = PMG::PBSConfig->new(); + if (defined($conf)) { + foreach my $remote (keys %{$conf->{ids}}) { + my $d = $conf->{ids}->{$remote}; + my $entry = { + remote => $remote, + server => $d->{server}, + datastore => $d->{datastore}, + }; + push @$res, $entry; + } + } + + return $res; + }}); + +__PACKAGE__->register_method ({ + name => 'remote_index', + path => '{remote}', + method => 'GET', + description => "Backup Job index.", + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { section => { type => 'string'} }, + }, + links => [ { rel => 'child', href => "{section}" } ], + }, + code => sub { + my ($param) = @_; + + my $result = [ + { section => 'snapshots' }, + { section => 'backup' }, + { section => 'restore' }, + { section => 'timer' }, + ]; + return $result; +}}); + +__PACKAGE__->register_method ({ + name => 'get_snapshots', + path => '{remote}/snapshots', + method => 'GET', + description => "Get snapshots stored on remote.", + proxyto => 'node', + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { + time => { type => 'string'}, + ctime => { type => 'string'}, + size => { type => 'integer'}, + }, + }, + links => [ { rel => 'child', href => "{time}" } ], + }, + code => sub { + my ($param) = @_; + + my $remote = $param->{remote}; + my $node = $param->{node}; + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + + my $snapshots = PMG::PBSTools::get_snapshots($remote_config, $remote); + my $res = []; + foreach my $item (@$snapshots) { + my $btype = $item->{"backup-type"}; + my $bid = $item->{"backup-id"}; + my $epoch = $item->{"backup-time"}; + my $size = $item->{size} // 1; + + next if !($btype eq 'host'); + next if !($bid eq $node); + + my $time = strftime("%FT%TZ", gmtime($epoch)); + + my $info = { + time => $time, + ctime => $epoch, + size => $size, + }; + + push @$res, $info; + } + + return $res; + }}); + +__PACKAGE__->register_method ({ + name => 'forget_snapshot', + path => '{remote}/snapshots/{time}', + method => 'DELETE', + description => "Forget a snapshot", + proxyto => 'node', + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + time => { + description => "Backup time in RFC 3399 format", + type => 'string', + }, + }, + }, + returns => {type => 'string' }, + code => sub { + my ($param) = @_; + + my $remote = $param->{remote}; + my $node = $param->{node}; + my $time = $param->{time}; + + my $snapshot = "host/$node/$time"; + + my $conf = PMG::PBSConfig->new(); + + my $rpcenv = PMG::RESTEnvironment->get(); + my $authuser = $rpcenv->get_user(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + + my $worker = sub { + eval { + PMG::PBSTools::forget_snapshot($remote_config, $remote, $snapshot); + }; + die "Forgetting backup failed: $@" if $@; + + return; + }; + + return $rpcenv->fork_worker('pbs_forget', undef, $authuser, $worker); + }}); + +__PACKAGE__->register_method ({ + name => 'run_backup', + path => '{remote}/backup', + method => 'POST', + description => "run backup", + proxyto => 'node', + protected => 1, + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { type => "string" }, + code => sub { + my ($param) = @_; + + my $rpcenv = PMG::RESTEnvironment->get(); + my $authuser = $rpcenv->get_user(); + + my $remote = $param->{remote}; + my $node = $param->{node}; + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + + my $backup_dir = "/var/lib/pmg/backup"; + my $currentdir = "${backup_dir}/current"; + + my $worker = sub { + my $upid = shift; + + print "starting update of current backup state\n"; + + PMG::Backup::pmg_backup($currentdir, $param->{statistic}); + my $pbs_opts = { + type => 'host', + id => $node, + pxarname => 'pmgbackup', + root => $currentdir, + }; + + PMG::PBSTools::backup_tree($remote_config, $remote, $pbs_opts); + + print "backup finished\n"; + + return; + }; + + return $rpcenv->fork_worker('pbs_backup', undef, $authuser, $worker); + + }}); + +__PACKAGE__->register_method ({ + name => 'restore', + path => '{remote}/restore', + method => 'POST', + description => "Restore the system configuration.", + permissions => { check => [ 'admin' ] }, + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + PMG::Backup::get_restore_options(), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + snapshot => {description=> "Backup-time to restore", + optional => 1, type => 'string' + }, + althost => {description => "hostname of backup snapshot", + optional => 1, type => 'string' + }, + }, + }, + returns => { type => "string" }, + code => sub { + my ($param) = @_; + + my $rpcenv = PMG::RESTEnvironment->get(); + my $authuser = $rpcenv->get_user(); + + my $remote = $param->{remote}; + my $node = $param->{althost} // $param->{node}; + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + + my $time = time; + my $dirname = "/tmp/proxrestore_$$.$time"; + + $param->{database} //= 1; + + die "nothing selected - please select what you want to restore (config or database?)\n" + if !($param->{database} || $param->{config}); + + my $pbs_opts = { + pxarname => 'pmgbackup', + target => $dirname, + }; + + $pbs_opts->{snapshot} = $param->{snapshot} // "host/$node"; + + my $worker = sub { + my $upid = shift; + + print "starting restore of $pbs_opts->{snapshot} from $remote\n"; + + PMG::PBSTools::restore_pxar($remote_config, $remote, $pbs_opts); + print "starting restore of PMG config\n"; + PMG::Backup::pmg_restore($dirname, $param->{database}, + $param->{config}, $param->{statistic}); + print "restore finished\n"; + + return; + }; + + return $rpcenv->fork_worker('pbs_restore', undef, $authuser, $worker); + }}); + +__PACKAGE__->register_method ({ + name => 'create_timer', + path => '{remote}/timer', + method => 'POST', + description => "Create backup schedule", + proxyto => 'node', + protected => 1, + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + schedule => { + description => "Schedule for the backup (OnCalendar setting of the systemd.timer)", + type => 'string', pattern => '[0-9a-zA-Z*.:,\-/ ]+', + default => 'daily', optional => 1, + }, + delay => { + description => "Randomized delay to add to the starttime (RandomizedDelaySec setting of the systemd.timer)", + type => 'string', pattern => '[0-9a-zA-Z. ]+', + default => 'daily', optional => 1, + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + my $remote = $param->{remote}; + my $schedule = $param->{schedule} // 'daily'; + my $delay = $param->{delay} // '5min'; + + my $conf = PMG::PBSConfig->new(); + + my $remote_config = $conf->{ids}->{$remote}; + die "PBS remote '$remote' does not exist\n" if !$remote_config; + + PMG::PBSTools::create_schedule($remote, $schedule, $delay); + + }}); + +__PACKAGE__->register_method ({ + name => 'delete_timer', + path => '{remote}/timer', + method => 'DELETE', + description => "Delete backup schedule", + proxyto => 'node', + protected => 1, + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + my $remote = $param->{remote}; + + PMG::PBSTools::delete_schedule($remote); + + }}); + +__PACKAGE__->register_method ({ + name => 'list_timer', + path => '{remote}/timer', + method => 'GET', + description => "Get timer specification", + proxyto => 'node', + protected => 1, + permissions => { check => [ 'admin', 'audit' ] }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + }, + }, + returns => { type => 'object', properties => { + remote => { + description => "Proxmox Backup Server ID.", + type => 'string', format => 'pve-configid', + }, + schedule => { + description => "Schedule for the backup (OnCalendar setting of the systemd.timer)", + type => 'string', pattern => '[0-9a-zA-Z*.:,\-/ ]+', + default => 'daily', optional => 1, + }, + delay => { + description => "Randomized delay to add to the starttime (RandomizedDelaySec setting of the systemd.timer)", + type => 'string', pattern => '[0-9a-zA-Z. ]+', + default => 'daily', optional => 1, + }, + unitfile => { + description => "unit file for the systemd.timer unit", + type => 'string', + }, + }}, + code => sub { + my ($param) = @_; + + my $remote = $param->{remote}; + + my $schedules = PMG::PBSTools::get_schedules(); + + my @data = grep {$_->{remote} eq $remote} @$schedules; + + die "Schedule for $remote not found!\n" if (scalar(@data != 1)); + my $res = $data[0]; + + return $res + }}); +1; -- 2.20.1