[PATCH manager 07/21] jobs/api: add basic host backup job logic

[Shannon Sterz <s.sterz@proxmox.com>]

adds a new job type "host-backup" that will back up certain host files
that are of particular concern to proxmox ve, this includes:

- everything below `/etc`
- interface pins: `/usr/local/lib/systemd/network/50-{pve,pmx}-*.link`
- pmxcfs database backup

to achieve a certain degree of consistency, the live backup feature of
pmxcfs is used to back up the database. for the rest of the files
included in a backup, snapshots are leveraged if the root file system
is detected to be either zfs or btrfs. for lvm-based systems, this
currently does not include any additional consistency measures.

users can add custom files and directories to a back-up via a
parameter. hooks can also be used to make the backup more versatile
and allow for improved consistency depending on the needs of users.

also, an api endpoint is added to trigger one-off host backups:

* POST /nodes/{node}/host-backup

Signed-off-by: Shannon Sterz <s.sterz@proxmox.com>
---
 PVE/API2/HostBackup.pm | 388 +++++++++++++++++++++++++++++++++++++++++
 PVE/API2/Makefile      |   1 +
 PVE/API2/Nodes.pm      |   7 +
 PVE/Jobs.pm            |   2 +
 PVE/Jobs/HostBackup.pm | 122 +++++++++++++
 PVE/Jobs/Makefile      |   5 +-
 6 files changed, 523 insertions(+), 2 deletions(-)
 create mode 100644 PVE/API2/HostBackup.pm
 create mode 100644 PVE/Jobs/HostBackup.pm

diff --git a/PVE/API2/HostBackup.pm b/PVE/API2/HostBackup.pm
new file mode 100644
index 000000000..9a0b0dc1d
--- /dev/null
+++ b/PVE/API2/HostBackup.pm
@@ -0,0 +1,388 @@
+package PVE::API2::HostBackup;
+
+use v5.36;
+
+use Carp;
+use Cwd qw(abs_path);
+use Encode qw(decode);
+use File::Path qw(make_path remove_tree);
+use JSON qw(decode_json encode_json);
+use Time::HiRes qw(usleep);
+
+use PVE::Cluster;
+use PVE::Cmd qw(run);
+use PVE::Exception qw(raise_param_exc);
+use PVE::INotify qw(nodename);
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::PBSClient;
+use PVE::Storage::PBSPlugin;
+use PVE::Storage;
+use PVE::Systemd;
+use PVE::VZDump;
+use PVE::pvecfg;
+
+use base qw(PVE::RESTHandler);
+
+my $host_backup_lock = "/run/pve/host-backup.lock";
+
+
+my sub prepare_backup($backup_name) {
+    my $snap_cmd = undef;
+    my $base_path = "/";
+
+    if (my $mntinfo = PVE::VZDump::get_mount_info("/")) {
+        if ($mntinfo->{fstype} eq "zfs") {
+            $snap_cmd = ['zfs', 'snapshot', $mntinfo->{device} . '@' . $backup_name];
+
+            # snapshots are mounted under `/.zfs/snapshots` unless snapdir is set to "disabled".
+            # the default is "hidden".
+            # https://openzfs.github.io/openzfs-docs/man/v2.4/7/zfsprops.7.html#snapdir
+            $base_path = "/.zfs/snapshot/$backup_name/";
+        } elsif ($mntinfo->{fstype} eq "btrfs") {
+            # check if snapshots directory exists, if not create one
+            mkdir "/.snapshots"
+                or $!{EEXIST}
+                or die "could not create snapshot folder - $!\n"
+                if !-d "/.snapshots";
+            $base_path = '/.snapshots/root@' . $backup_name . '/';
+            $snap_cmd = [
+                'btrfs', '-q', 'subvolume', 'snapshot', '-r', '--', '/', $base_path,
+            ];
+        }
+    }
+
+    my $result = PVE::Cluster::cfs_live_backup_database();
+    my $progress = $result->{'progress'};
+
+    while ($progress->{'in-progress'}) {
+        print("Backing up pmxcfs $progress->{remaining} of $progress->{total} pages remain...\n");
+        usleep(500_000); # Sleep for half a second to avoid spamming pmxcfs which could pin a CPU.
+        $progress = PVE::Cluster::cfs_live_backup_progress();
+    }
+
+    my $db_file = "var/lib/pve-cluster/backup/" . $result->{'file'};
+
+    die "no pmxcfs backup was created, aborting backup...\n" if !-f "/$db_file";
+
+    if (defined($snap_cmd)) {
+        eval { run($snap_cmd, errmsg => "could not create root file system snapshot"); };
+        die $@ if $@;
+    }
+
+    return ($base_path, $db_file);
+}
+
+my sub do_backup : prototype($$\@$$) ($pbs, $base_path, $files, $backup_name, $backup_target) {
+    croak "backup directory has not been created yet, aborting...\n" if !-d $backup_target;
+
+    my $errors = "";
+    my $rsync_cmd = ['rsync', '-aqXA', '--relative', '--one-file-system'];
+
+    my $err_func = sub {
+        my ($line) = @_;
+        $errors .= decode('UTF-8', $line);
+    };
+
+    foreach my $to_backup (@{$files}) {
+        # skip files that don't exist without displaying an error
+        next if !-e "/$to_backup";
+        # add a `./` here to only copy necessary relative paths
+        push @$rsync_cmd, $base_path . './' . $to_backup;
+    }
+
+    push @$rsync_cmd, $backup_target;
+    run($rsync_cmd, errfunc => $err_func);
+
+    my @auto_packages = ();
+    run(
+        ['apt-mark', 'showauto'],
+        errmsg => "could not get automatically installed packages",
+        outfunc => sub {
+            my ($line) = @_;
+            push @auto_packages, decode('UTF-8', $line);
+        },
+    );
+
+    my @manual_packages = ();
+    run(
+        ['apt-mark', 'showmanual'],
+        errmsg => "could not get manually installed packages",
+        outfunc => sub {
+            my ($line) = @_;
+            push @manual_packages, decode('UTF-8', $line);
+        },
+    );
+
+    my $index = {
+        'backup-name' => $backup_name,
+        'version' => {
+            'proxmox-ve' => PVE::pvecfg::version_info(),
+        },
+        'package-information' => {
+            'automatically-installed' => \@auto_packages,
+            'manually-installed' => \@manual_packages,
+        },
+    };
+
+    my $os_release = undef;
+
+    if (-f '/etc/os-release') {
+        # On our current Debian Trixie releases this is a symlink to
+        # `/usr/lib/os-release`, but `man 5 os-release` specifies that
+        # `/etc/os-release` takes precedence, so adhere to that.
+        $os_release = PVE::Tools::file_get_contents('/etc/os-release');
+    } elsif (-f '/usr/lib/os-release') {
+        $os_release = PVE::Tools::file_get_contents('/usr/lib/os-release');
+    }
+
+    $index->{'version'}->{'os-release'} = PVE::Systemd::parse_os_release($os_release)
+        if defined($os_release);
+
+    if ($errors ne "") {
+        $index->{'errors'} = $errors;
+    }
+
+    my $nodename = nodename();
+    PVE::Tools::file_set_contents($backup_target . '/backup-index.json', encode_json($index));
+    $pbs->backup_fs_tree($backup_target, $nodename, 'pve-backup');
+}
+
+my sub cleanup_backup($backup_name, $backup_target) {
+    eval { remove_tree($backup_target, { safe => 1 }) if -d $backup_target; };
+    warn $@ if $@;
+
+    my $snap_cmd = undef;
+
+    if (my $mntinfo = PVE::VZDump::get_mount_info("/")) {
+        if ($mntinfo->{fstype} eq "zfs") {
+            $snap_cmd = ['zfs', 'destroy', $mntinfo->{device} . '@' . $backup_name];
+        } elsif ($mntinfo->{fstype} eq "btrfs") {
+            $snap_cmd = [
+                'btrfs', '-q', 'subvolume', 'delete', '--', '/.snapshots/root@' . $backup_name,
+            ];
+        }
+    }
+
+    if (defined($snap_cmd)) {
+        run(
+            $snap_cmd, errmsg => "could not clean up root file system snapshot",
+        );
+    }
+}
+
+my sub run_hook_script : prototype($$$;\%) ($hooks, $phase, $storage_cfg, $payload = undef) {
+    return if !defined($hooks);
+
+    my ($path, undef, $type) = PVE::Storage::path($storage_cfg, $hooks);
+
+    croak "Error: The hook script '$hooks' is not a snippet.\n" if $type ne "snippets";
+    croak "Error: The hook script '$hooks' does not exist or is not a file.\n" if !-f $path;
+    croak "Error: The hook script '$hooks' is not executable.\n" if !-x $path;
+
+    my $json = '';
+    my $json_payload = '';
+    my $cmd = undef;
+
+    if (defined($payload)) {
+        $json_payload = encode_json($payload);
+    }
+
+    run(
+        [$path, $phase],
+        errmsg => "error while executing hook script '$hooks' in phase '$phase'",
+        input => $json_payload,
+        outfunc => sub {
+            my ($line) = @_;
+            $json .= decode('UTF-8', $line) . "\n";
+        },
+    );
+
+    return if !$json;
+    return decode_json($json);
+}
+
+my sub exec_host_backup : prototype(\%) ($conf) {
+    my $ctime = time();
+
+    my $backup_name = "host-backup-$ctime";
+    my $base_path = undef;
+    my $db_backup_file = undef;
+
+    croak "no storage defined, can't carry out host backup\n" if !defined($conf->{storage});
+
+    my $storeid = $conf->{storage};
+    my $cfg = PVE::Storage::config();
+    my $scfg = PVE::Storage::storage_check_enabled($cfg, $storeid, undef, 1);
+
+    # croak here since providing a viable storage is the responsibility of the caller
+    croak "storage is not enabled or not available on this host\n" if !defined($scfg);
+    croak "currently only proxmox backup server is supported for host backups\n"
+        if $scfg->{type} ne PVE::Storage::PBSPlugin::type();
+
+    my $pbs = PVE::PBSClient->new($scfg, $storeid);
+
+    PVE::Tools::lock_file(
+        $host_backup_lock,
+        undef,
+        sub {
+            eval {
+                ($base_path, $db_backup_file) = prepare_backup($backup_name);
+
+                if (my $result = run_hook_script($conf->{"hooks"}, "job-start", $cfg)) {
+                    if (
+                        defined($result->{"base-path"})
+                        && -d $result->{"base-path"}
+                    ) {
+                        print "Using custom base path from hook script: "
+                            . $result->{"base-path"} . "\n";
+                        $base_path = $result->{"base-path"};
+                    }
+                }
+            };
+
+            my $err = $@;
+            my $backup_target = "/run/pve/host-backup/$backup_name";
+
+            if (!$err) {
+                my $backup_info = {
+                    "backup-name" => $backup_name,
+                    "backup-target" => $backup_target,
+                };
+
+                eval {
+                    make_path($backup_target, { mode => 0700 });
+
+                    run_hook_script($conf->{"hooks"}, "backup-start", $cfg, %$backup_info);
+
+                    my @files = ("etc/", $db_backup_file);
+
+                    my @link_files =
+                        glob("$base_path/usr/local/lib/systemd/network/50-{pve,pmx}-*.link");
+
+                    foreach my $link (@link_files) {
+                        # strip the base path, `do_backup` expects relative paths.
+                        # this should also un-taint the string
+                        if ($link =~ m/^\Q$base_path\E\/(.*)$/) {
+                            push @files, $1;
+                        }
+                    }
+
+                    if (defined($conf->{"additional-files"})) {
+                        my @additional_files =
+                            PVE::Tools::split_list($conf->{"additional-files"});
+
+                        foreach my $additional (@additional_files) {
+                            my $abs_path = abs_path($additional);
+                            # untaint and strip first `/`
+                            if ($abs_path =~ m/\/(.*)/) {
+                                push @files, $1;
+                            }
+                        }
+                    }
+
+                    do_backup($pbs, $base_path, @files, $backup_name, $backup_target);
+                    run_hook_script($conf->{"hooks"}, "backup-end", $cfg, %$backup_info);
+                };
+                warn "error while creating host backup: $@\n" if $@;
+                $err ||= $@;
+            }
+
+            eval { cleanup_backup($backup_name, $backup_target) };
+            warn "error while cleaning up backup: $@\n" if $@;
+            $err ||= $@;
+
+            eval { run_hook_script($conf->{"hooks"}, "job-end", $cfg); };
+            warn "could not run 'job-end' hook: $@\n" if $@;
+            $err ||= $@;
+
+            die "host backup failed: $err\n" if $err;
+        },
+    );
+    die $@ if $@;
+}
+
+sub complete_proxmox_backup_storage(@) {
+    my $cfg = PVE::Storage::config();
+    my $nodename = PVE::INotify::nodename();
+    my $ids = $cfg->{ids};
+    my $res = [];
+
+    foreach my $storageid (keys %$ids) {
+        my $scfg = PVE::Storage::storage_check_enabled($cfg, $storageid, $nodename, 1);
+        next if $scfg->{type} ne PVE::Storage::PBSPlugin::type();
+        push @$res, $storageid;
+    }
+
+    return $res;
+}
+
+__PACKAGE__->register_method({
+    name => 'create_backup',
+    path => '',
+    method => 'POST',
+    proxyto => 'node',
+    protected => 1,
+    permissions => {
+        description =>
+            "The user needs 'Sys.Console' permissions on '/' and 'Datastore.AllocateSpace' "
+            . "permissions on the target storage. The parameters 'hooks' and 'additional-files' are"
+            . "further restricted to 'root\@pam'.",
+        check => [
+            'and',
+            ['perm', '/', ['Sys.Console']],
+            ['perm', '/storage/{storage}', ['Datastore.AllocateSpace']],
+        ],
+
+    },
+    description => "Create a new host backup.",
+    parameters => {
+        additionalProperties => 0,
+        properties => {
+            node => get_standard_option('pve-node'),
+            storage => get_standard_option(
+                'pve-storage-id',
+                {
+                    description => "Store resulting file to this storage.",
+                    completion => \&complete_proxmox_backup_storage,
+                    optional => 0,
+                },
+            ),
+            hooks => {
+                type => 'string',
+                description => "Use specified hook script.",
+                optional => 1,
+            },
+            'additional-files' => {
+                description =>
+                    "Additional files or directories that should be included in the backup."
+                    . " Separated by a comma.",
+                type => 'string',
+                optional => 1,
+            },
+        },
+    },
+    returns => { type => 'string' },
+    code => sub($param) {
+        my $rpcenv = PVE::RPCEnvironment::get();
+        my $user = $rpcenv->get_user();
+
+        if ($user ne "root\@pam") {
+            if (defined($param->{hooks})) {
+                raise_param_exc({ hooks => "Only root may set the hooks option." });
+            }
+
+            if (defined($param->{"additional-files"})) {
+                raise_param_exc({
+                    "additional-files" => "Only root may set the additional-files option." });
+            }
+        }
+
+        my $worker = sub {
+            exec_host_backup(%$param);
+        };
+
+        return $rpcenv->fork_worker('host-backup', $param->{node}, $user, $worker);
+    },
+});
+
+1;
diff --git a/PVE/API2/Makefile b/PVE/API2/Makefile
index 97f1cc202..1cd72a594 100644
--- a/PVE/API2/Makefile
+++ b/PVE/API2/Makefile
@@ -14,6 +14,7 @@ PERLSOURCE = 			\
 	Cluster.pm		\
 	HAConfig.pm		\
 	Hardware.pm		\
+	HostBackup.pm		\
 	Network.pm		\
 	NodeConfig.pm		\
 	Nodes.pm		\
diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
index 2ca3244da..0879443fc 100644
--- a/PVE/API2/Nodes.pm
+++ b/PVE/API2/Nodes.pm
@@ -49,6 +49,7 @@ use PVE::API2::Certificates;
 use PVE::API2::Disks;
 use PVE::API2::Firewall::Host;
 use PVE::API2::Hardware;
+use PVE::API2::HostBackup;
 use PVE::API2::LXC::Status;
 use PVE::API2::LXC;
 use PVE::API2::Network;
@@ -204,6 +205,11 @@ __PACKAGE__->register_method({
     path => 'sdn',
 });
 
+__PACKAGE__->register_method({
+    subclass => "PVE::API2::HostBackup",
+    path => 'host-backup',
+});
+
 __PACKAGE__->register_method({
     name => 'index',
     path => '',
@@ -269,6 +275,7 @@ __PACKAGE__->register_method({
             { name => 'vncshell' },
             { name => 'vzdump' },
             { name => 'wakeonlan' },
+            { name => 'host-backup' },
         ];
 
         return $result;
diff --git a/PVE/Jobs.pm b/PVE/Jobs.pm
index 40caf257d..097cee5c6 100644
--- a/PVE/Jobs.pm
+++ b/PVE/Jobs.pm
@@ -9,9 +9,11 @@ use PVE::Job::Registry;
 use PVE::Jobs::VZDump;
 use PVE::Jobs::RealmSync;
 use PVE::Tools;
+use PVE::Jobs::HostBackup;
 
 PVE::Jobs::VZDump->register();
 PVE::Jobs::RealmSync->register();
+PVE::Jobs::HostBackup->register();
 PVE::Job::Registry->init();
 
 cfs_register_file(
diff --git a/PVE/Jobs/HostBackup.pm b/PVE/Jobs/HostBackup.pm
new file mode 100644
index 000000000..787b19547
--- /dev/null
+++ b/PVE/Jobs/HostBackup.pm
@@ -0,0 +1,122 @@
+package PVE::Jobs::HostBackup;
+
+use v5.36;
+
+use PVE::API2::HostBackup;
+use PVE::Cluster;
+use PVE::INotify qw(nodename);
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::SafeSyslog;
+
+use parent qw(PVE::Job::Registry);
+
+sub type($) {
+    return 'host-backup';
+}
+
+my $props = {
+    nodes => get_standard_option(
+        'pve-node-list',
+        {
+            description => "List of nodes for which the storage configuration applies.",
+            optional => 1,
+        },
+    ),
+    hooks => {
+        description => "Use specified hook script.",
+        type => 'string',
+        optional => 1,
+    },
+    'additional-files' => {
+        description => "Additional files or directories that should be included in the backup."
+            . " Separated by a comma.",
+        type => 'string',
+        optional => 1,
+    },
+};
+
+sub properties($) {
+    return $props;
+}
+
+sub options($) {
+    my $options = {
+        comment => { optional => 1 },
+        enabled => { optional => 1 },
+        'repeat-missed' => { optional => 1 },
+        schedule => {},
+        storage => {},
+    };
+
+    foreach my $opt (keys %$props) {
+        if ($props->{$opt}->{optional}) {
+            $options->{$opt} = { optional => 1 };
+        } else {
+            $options->{$opt} = {};
+        }
+    }
+
+    return $options;
+}
+
+sub decode_value($class, $type, $key, $value) {
+    return $value;
+}
+
+sub encode_value($class, $type, $key, $value) {
+    return $value;
+}
+
+# Returns the create Schema of a host backup job.
+#
+# Override the Registry's base implementation as SectionConfig would otherwise return any option
+# registered for any job. This is due to all jobs sharing a base class that maintains globally all
+# fields for the jobs.cfg section config. However, when creating a host backup job, many options from
+# either VZDump nor the realm sync jobs apply. So filter them out here and only return properties
+# relevant for the host backup.
+sub createSchema($class) {
+    my $to_return = {
+        additionalProperties => 0,
+        properties => {},
+    };
+
+    my $opts = $class->options();
+    my $super_schema = $class->SUPER::createSchema();
+
+    foreach my $opt (keys %$opts) {
+        $to_return->{properties}->{$opt} = $super_schema->{properties}->{$opt};
+    }
+
+    return $to_return;
+}
+
+sub run($class, $conf, $job_id, $schedule) {
+    my $nodename = nodename();
+
+    # the jobs framework allows specifying whether a job should run on a single
+    # node. in the context of host backups, jobs are assigned to a subset of
+    # nodes. simply report 'OK' if this node is not part of the job to mark the
+    # job as "done".
+    if (defined($conf->{nodes})) {
+        my @nodes = PVE::Tools::split_list($conf->{nodes});
+        if (!grep(/^$nodename$/, @nodes)) {
+            return 'OK';
+        }
+    }
+
+    my $job_conf = {
+        node => $nodename,
+        storage => $conf->{'storage'},
+    };
+
+    foreach my $opt (keys %$conf) {
+        $job_conf->{$opt} = $conf->{$opt} if defined($props->{$opt});
+    }
+
+    # this is a scheduling-only parameter, so remove it here as the api won't allow it.
+    delete $job_conf->{nodes} if $job_conf->{nodes};
+
+    return PVE::API2::HostBackup->create_backup($job_conf);
+}
+
+1;
diff --git a/PVE/Jobs/Makefile b/PVE/Jobs/Makefile
index 11aed0d19..9c4b702b0 100644
--- a/PVE/Jobs/Makefile
+++ b/PVE/Jobs/Makefile
@@ -1,7 +1,8 @@
 include ../../defines.mk
 
-PERLSOURCE =   \
-	VZDump.pm \
+PERLSOURCE =		\
+	HostBackup.pm	\
+	VZDump.pm		\
 
 all:
 
-- 
2.47.3