public inbox for pmg-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Stoiko Ivanov <s.ivanov@proxmox.com>
To: pmg-devel@lists.proxmox.com
Subject: [pmg-devel] [RFC pmg-api 11/12] Add API2 module for per-node backups to PBS
Date: Mon, 19 Oct 2020 21:02:08 +0200	[thread overview]
Message-ID: <20201019190209.11495-12-s.ivanov@proxmox.com> (raw)
In-Reply-To: <20201019190209.11495-1-s.ivanov@proxmox.com>

The module adds API2 methods for:

* creating/restoring/listing/forgetting backups to a configured PBS remote
* creating backup schedules (using systemd-timers)

Signed-off-by: Stoiko Ivanov <s.ivanov@proxmox.com>
---
 src/Makefile            |   1 +
 src/PMG/API2/Nodes.pm   |   7 +
 src/PMG/API2/PBS/Job.pm | 467 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 475 insertions(+)
 create mode 100644 src/PMG/API2/PBS/Job.pm

diff --git a/src/Makefile b/src/Makefile
index e1546a8..b978cfa 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -138,6 +138,7 @@ LIBSOURCES =				\
 	PMG/API2/Statistics.pm		\
 	PMG/API2/MailTracker.pm		\
 	PMG/API2/Backup.pm		\
+	PMG/API2/PBS/Job.pm		\
 	PMG/API2/PBS/Remote.pm		\
 	PMG/API2/Nodes.pm		\
 	PMG/API2/Postfix.pm		\
diff --git a/src/PMG/API2/Nodes.pm b/src/PMG/API2/Nodes.pm
index 96aa146..259f8f3 100644
--- a/src/PMG/API2/Nodes.pm
+++ b/src/PMG/API2/Nodes.pm
@@ -26,6 +26,7 @@ use PMG::API2::SpamAssassin;
 use PMG::API2::Postfix;
 use PMG::API2::MailTracker;
 use PMG::API2::Backup;
+use PMG::API2::PBS::Job;
 
 use base qw(PVE::RESTHandler);
 
@@ -79,6 +80,11 @@ __PACKAGE__->register_method ({
     path => 'backup',
 });
 
+__PACKAGE__->register_method ({
+    subclass => "PMG::API2::PBS::Job",
+    path => 'pbs',
+});
+
 __PACKAGE__->register_method ({
     name => 'index',
     path => '',
@@ -105,6 +111,7 @@ __PACKAGE__->register_method ({
 	my $result = [
 	    { name => 'apt' },
 	    { name => 'backup' },
+	    { name => 'pbs' },
 	    { name => 'clamav' },
 	    { name => 'spamassassin' },
 	    { name => 'postfix' },
diff --git a/src/PMG/API2/PBS/Job.pm b/src/PMG/API2/PBS/Job.pm
new file mode 100644
index 0000000..8fa3a19
--- /dev/null
+++ b/src/PMG/API2/PBS/Job.pm
@@ -0,0 +1,467 @@
+package PMG::API2::PBS::Job;
+
+use strict;
+use warnings;
+
+use POSIX qw(strftime);
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RESTHandler;
+use PVE::SafeSyslog;
+use PVE::Tools qw(extract_param);
+
+use PMG::RESTEnvironment;
+use PMG::Backup;
+use PMG::PBSTools;
+use PMG::PBSConfig;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+    name => 'list',
+    path => '',
+    method => 'GET',
+    description => "List all configured Proxmox Backup Server jobs.",
+    permissions => { check => [ 'admin', 'audit' ] },
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	},
+    },
+    returns => {
+        type => "array",
+        items => PMG::PBSConfig->createSchema(1),
+        links => [ { rel => 'child', href => "{remote}" } ],
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $res = [];
+
+	my $conf = PMG::PBSConfig->new();
+	if (defined($conf)) {
+	    foreach my $remote (keys %{$conf->{ids}}) {
+		my $d = $conf->{ids}->{$remote};
+		my $entry = {
+		    remote => $remote,
+		    server => $d->{server},
+		    datastore => $d->{datastore},
+		};
+		push @$res, $entry;
+	    }
+	}
+
+	return $res;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'remote_index',
+    path => '{remote}',
+    method => 'GET',
+    description => "Backup Job index.",
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	},
+    },
+    returns => {
+	type => 'array',
+	items => {
+	    type => "object",
+	    properties => { section => { type => 'string'} },
+	},
+	links => [ { rel => 'child', href => "{section}" } ],
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $result = [
+	    { section => 'snapshots' },
+	    { section => 'backup' },
+	    { section => 'restore' },
+	    { section => 'timer' },
+	];
+	return $result;
+}});
+
+__PACKAGE__->register_method ({
+    name => 'get_snapshots',
+    path => '{remote}/snapshots',
+    method => 'GET',
+    description => "Get snapshots stored on remote.",
+    proxyto => 'node',
+    permissions => { check => [ 'admin', 'audit' ] },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	},
+    },
+    returns => {
+	type => 'array',
+	items => {
+	    type => "object",
+	    properties => {
+		time => { type => 'string'},
+		ctime => { type => 'string'},
+		size => { type => 'integer'},
+	    },
+	},
+	links => [ { rel => 'child', href => "{time}" } ],
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $remote = $param->{remote};
+	my $node = $param->{node};
+
+	my $conf = PMG::PBSConfig->new();
+
+	my $remote_config = $conf->{ids}->{$remote};
+	die "PBS remote '$remote' does not exist\n" if !$remote_config;
+
+	my $snapshots = PMG::PBSTools::get_snapshots($remote_config, $remote);
+	my $res = [];
+	foreach my $item (@$snapshots) {
+	    my $btype = $item->{"backup-type"};
+	    my $bid = $item->{"backup-id"};
+	    my $epoch = $item->{"backup-time"};
+	    my $size = $item->{size} // 1;
+
+	    next if !($btype eq 'host');
+	    next if !($bid eq $node);
+
+	    my $time = strftime("%FT%TZ", gmtime($epoch));
+
+	    my $info = {
+		time => $time,
+		ctime => $epoch,
+		size => $size,
+	    };
+
+	    push @$res, $info;
+	}
+
+	return $res;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'forget_snapshot',
+    path => '{remote}/snapshots/{time}',
+    method => 'DELETE',
+    description => "Forget a snapshot",
+    proxyto => 'node',
+    permissions => { check => [ 'admin', 'audit' ] },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	    time => {
+		description => "Backup time in RFC 3399 format",
+		type => 'string',
+	    },
+	},
+    },
+    returns => {type => 'string' },
+    code => sub {
+	my ($param) = @_;
+
+	my $remote = $param->{remote};
+	my $node = $param->{node};
+	my $time = $param->{time};
+
+	my $snapshot = "host/$node/$time";
+
+	my $conf = PMG::PBSConfig->new();
+
+	my $rpcenv = PMG::RESTEnvironment->get();
+	my $authuser = $rpcenv->get_user();
+
+	my $remote_config = $conf->{ids}->{$remote};
+	die "PBS remote '$remote' does not exist\n" if !$remote_config;
+
+	my $worker = sub {
+	    eval {
+		PMG::PBSTools::forget_snapshot($remote_config, $remote, $snapshot);
+	    };
+	    die "Forgetting backup failed: $@" if $@;
+
+	    return;
+	};
+
+	return $rpcenv->fork_worker('pbs_forget', undef, $authuser, $worker);
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'run_backup',
+    path => '{remote}/backup',
+    method => 'POST',
+    description => "run backup",
+    proxyto => 'node',
+    protected => 1,
+    permissions => { check => [ 'admin', 'audit' ] },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	},
+    },
+    returns => { type => "string" },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PMG::RESTEnvironment->get();
+	my $authuser = $rpcenv->get_user();
+
+	my $remote = $param->{remote};
+	my $node = $param->{node};
+
+	my $conf = PMG::PBSConfig->new();
+
+	my $remote_config = $conf->{ids}->{$remote};
+	die "PBS remote '$remote' does not exist\n" if !$remote_config;
+
+	my $backup_dir = "/var/lib/pmg/backup";
+	my $currentdir = "${backup_dir}/current";
+
+	my $worker = sub {
+	    my $upid = shift;
+
+	    print "starting update of current backup state\n";
+
+	    PMG::Backup::pmg_backup($currentdir, $param->{statistic});
+	    my $pbs_opts = {
+		type => 'host',
+		id => $node,
+		pxarname => 'pmgbackup',
+		root => $currentdir,
+	    };
+
+	    PMG::PBSTools::backup_tree($remote_config, $remote, $pbs_opts);
+
+	    print "backup finished\n";
+
+	    return;
+	};
+
+	return $rpcenv->fork_worker('pbs_backup', undef, $authuser, $worker);
+
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'restore',
+    path => '{remote}/restore',
+    method => 'POST',
+    description => "Restore the system configuration.",
+    permissions => { check => [ 'admin' ] },
+    proxyto => 'node',
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    PMG::Backup::get_restore_options(),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	    snapshot => {description=> "Backup-time to restore",
+		optional => 1, type => 'string'
+	    },
+	    althost => {description => "hostname of backup snapshot",
+		optional => 1, type => 'string'
+	    },
+	},
+    },
+    returns => { type => "string" },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PMG::RESTEnvironment->get();
+	my $authuser = $rpcenv->get_user();
+
+	my $remote = $param->{remote};
+	my $node = $param->{althost} // $param->{node};
+
+	my $conf = PMG::PBSConfig->new();
+
+	my $remote_config = $conf->{ids}->{$remote};
+	die "PBS remote '$remote' does not exist\n" if !$remote_config;
+
+	my $time = time;
+	my $dirname = "/tmp/proxrestore_$$.$time";
+
+	$param->{database} //= 1;
+
+	die "nothing selected - please select what you want to restore (config or database?)\n"
+	    if !($param->{database} || $param->{config});
+
+	my $pbs_opts = {
+	    pxarname => 'pmgbackup',
+	    target => $dirname,
+	};
+
+	$pbs_opts->{snapshot} = $param->{snapshot} // "host/$node";
+
+	my $worker = sub {
+	    my $upid = shift;
+
+	    print "starting restore of $pbs_opts->{snapshot} from $remote\n";
+
+	    PMG::PBSTools::restore_pxar($remote_config, $remote, $pbs_opts);
+	    print "starting restore of PMG config\n";
+	    PMG::Backup::pmg_restore($dirname, $param->{database},
+		 $param->{config}, $param->{statistic});
+	    print "restore finished\n";
+
+	    return;
+	};
+
+	return $rpcenv->fork_worker('pbs_restore', undef, $authuser, $worker);
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'create_timer',
+    path => '{remote}/timer',
+    method => 'POST',
+    description => "Create backup schedule",
+    proxyto => 'node',
+    protected => 1,
+    permissions => { check => [ 'admin', 'audit' ] },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	    schedule => {
+		description => "Schedule for the backup (OnCalendar setting of the systemd.timer)",
+		type => 'string', pattern => '[0-9a-zA-Z*.:,\-/ ]+',
+		default => 'daily', optional => 1,
+	    },
+	    delay => {
+		description => "Randomized delay to add to the starttime (RandomizedDelaySec setting of the systemd.timer)",
+		type => 'string', pattern => '[0-9a-zA-Z. ]+',
+		default => 'daily', optional => 1,
+	    },
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	my $remote = $param->{remote};
+	my $schedule = $param->{schedule} // 'daily';
+	my $delay = $param->{delay} // '5min';
+
+	my $conf = PMG::PBSConfig->new();
+
+	my $remote_config = $conf->{ids}->{$remote};
+	die "PBS remote '$remote' does not exist\n" if !$remote_config;
+
+	PMG::PBSTools::create_schedule($remote, $schedule, $delay);
+
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'delete_timer',
+    path => '{remote}/timer',
+    method => 'DELETE',
+    description => "Delete backup schedule",
+    proxyto => 'node',
+    protected => 1,
+    permissions => { check => [ 'admin', 'audit' ] },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	my $remote = $param->{remote};
+
+	PMG::PBSTools::delete_schedule($remote);
+
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'list_timer',
+    path => '{remote}/timer',
+    method => 'GET',
+    description => "Get timer specification",
+    proxyto => 'node',
+    protected => 1,
+    permissions => { check => [ 'admin', 'audit' ] },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	},
+    },
+    returns => { type => 'object', properties => {
+	    remote => {
+		description => "Proxmox Backup Server ID.",
+		type => 'string', format => 'pve-configid',
+	    },
+	    schedule => {
+		description => "Schedule for the backup (OnCalendar setting of the systemd.timer)",
+		type => 'string', pattern => '[0-9a-zA-Z*.:,\-/ ]+',
+		default => 'daily', optional => 1,
+	    },
+	    delay => {
+		description => "Randomized delay to add to the starttime (RandomizedDelaySec setting of the systemd.timer)",
+		type => 'string', pattern => '[0-9a-zA-Z. ]+',
+		default => 'daily', optional => 1,
+	    },
+	    unitfile => {
+		description => "unit file for the systemd.timer unit",
+		type => 'string',
+	    },
+	}},
+    code => sub {
+	my ($param) = @_;
+
+	my $remote = $param->{remote};
+
+	my $schedules = PMG::PBSTools::get_schedules();
+
+	my @data = grep {$_->{remote} eq $remote} @$schedules;
+
+	die "Schedule for $remote not found!\n" if (scalar(@data != 1));
+	my $res = $data[0];
+
+	return $res
+    }});
+1;
-- 
2.20.1





  parent reply	other threads:[~2020-10-19 19:02 UTC|newest]

Thread overview: 14+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-10-19 19:01 [pmg-devel] [RFC pmg-api 00/12] POC PBS integration Stoiko Ivanov
2020-10-19 19:01 ` [pmg-devel] [RFC pmg-api 01/12] drop left-over commented out code Stoiko Ivanov
2020-10-19 19:01 ` [pmg-devel] [RFC pmg-api 02/12] Backup: split backup creation and creating tar Stoiko Ivanov
2020-10-20  5:43   ` Dietmar Maurer
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 03/12] Restore: optionally restore from directory Stoiko Ivanov
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 04/12] Backup: push restore options to PMG::Backup Stoiko Ivanov
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 05/12] debian: add dependency on proxmox-backup-client Stoiko Ivanov
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 06/12] add helper module for handling PBS Integration Stoiko Ivanov
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 07/12] PBSTools: add methods for managing backups Stoiko Ivanov
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 08/12] PBSTools: add systemd-timer helpers Stoiko Ivanov
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 09/12] add initial SectionConfig for pbs Stoiko Ivanov
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 10/12] Add API2 module for PBS configuration Stoiko Ivanov
2020-10-19 19:02 ` Stoiko Ivanov [this message]
2020-10-19 19:02 ` [pmg-devel] [RFC pmg-api 12/12] pbs-integration: add CLI calls to pmgbackup Stoiko Ivanov

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20201019190209.11495-12-s.ivanov@proxmox.com \
    --to=s.ivanov@proxmox.com \
    --cc=pmg-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal