Re: [pve-devel] [POC storage v3 15/34] WIP Borg plugin

["Fabian Grünbichler" <f.gruenbichler@proxmox.com>]

On November 7, 2024 5:51 pm, Fiona Ebner wrote:
> Archive names start with the guest type and ID and then the same
> timestamp format as PBS.
> 
> Container archives have the following structure:
> guest.config
> firewall.config
> filesystem/ # containing the whole filesystem structure
> 
> VM archives have the following structure
> guest.config
> firewall.config
> volumes/ # containing a raw file for each device
> 
> A bindmount (respectively symlinks) are used to achieve this
> structure, because Borg doesn't seem to support renaming on-the-fly.
> (Prefix stripping via the "slashdot hack" would have helped slightly,
> but is only in Borg >= 1.4
> https://github.com/borgbackup/borg/actions/runs/7967940995)
> 
> NOTE: Bandwidth limit is not yet honored and the task size is not
> calculated yet. Discard for VM backups would also be nice to have, but
> it's not entirely clear how (parsing progress and discarding according
> to that is one idea). There is no dirty bitmap support, not sure if
> that is feasible to add.
> 
> Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
> ---
> 
> Changes in v3:
> * make SSH work.
> * adapt to API changes, i.e. config as raw data and user namespace
>   execution context for containers.
> 
>  src/PVE/API2/Storage/Config.pm         |   2 +-
>  src/PVE/BackupProvider/Plugin/Borg.pm  | 439 ++++++++++++++++++
>  src/PVE/BackupProvider/Plugin/Makefile |   2 +-
>  src/PVE/Storage.pm                     |   2 +
>  src/PVE/Storage/BorgBackupPlugin.pm    | 595 +++++++++++++++++++++++++
>  src/PVE/Storage/Makefile               |   1 +
>  6 files changed, 1039 insertions(+), 2 deletions(-)
>  create mode 100644 src/PVE/BackupProvider/Plugin/Borg.pm
>  create mode 100644 src/PVE/Storage/BorgBackupPlugin.pm
> 
> diff --git a/src/PVE/API2/Storage/Config.pm b/src/PVE/API2/Storage/Config.pm
> index e04b6ab..1cbf09d 100755
> --- a/src/PVE/API2/Storage/Config.pm
> +++ b/src/PVE/API2/Storage/Config.pm
> @@ -190,7 +190,7 @@ __PACKAGE__->register_method ({
>  	return &$api_storage_config($cfg, $param->{storage});
>      }});
>  
> -my $sensitive_params = [qw(password encryption-key master-pubkey keyring)];
> +my $sensitive_params = [qw(password encryption-key master-pubkey keyring ssh-key)];
>  
>  __PACKAGE__->register_method ({
>      name => 'create',
> diff --git a/src/PVE/BackupProvider/Plugin/Borg.pm b/src/PVE/BackupProvider/Plugin/Borg.pm
> new file mode 100644
> index 0000000..7bb3ae3
> --- /dev/null
> +++ b/src/PVE/BackupProvider/Plugin/Borg.pm
> @@ -0,0 +1,439 @@
> +package PVE::BackupProvider::Plugin::Borg;
> +
> +use strict;
> +use warnings;
> +
> +use File::chdir;
> +use File::Basename qw(basename);
> +use File::Path qw(make_path remove_tree);
> +use Net::IP;
> +use POSIX qw(strftime);
> +
> +use PVE::Tools;
> +
> +# ($vmtype, $vmid, $time_string)
> +our $ARCHIVE_RE_3 = qr!^pve-(lxc|qemu)-([0-9]+)-([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$!;
> +
> +sub archive_name {
> +    my ($vmtype, $vmid, $backup_time) = @_;
> +
> +    return "pve-${vmtype}-${vmid}-" . strftime("%FT%TZ", gmtime($backup_time));
> +}
> +
> +# remove_tree can be very verbose by default, do explicit error handling and limit to one message
> +my sub _remove_tree {
> +    my ($path) = @_;
> +
> +    remove_tree($path, { error => \my $err });
> +    if ($err && @$err) { # empty array if no error
> +	for my $diag (@$err) {
> +	    my ($file, $message) = %$diag;
> +	    die "cannot remove_tree '$path': $message\n" if $file eq '';
> +	    die "cannot remove_tree '$path': unlinking $file failed - $message\n";
> +	}
> +    }
> +}
> +
> +my sub prepare_run_dir {
> +    my ($archive, $operation) = @_;
> +
> +    my $run_dir = "/run/pve-storage-borg-plugin/${archive}.${operation}";
> +    _remove_tree($run_dir);
> +    make_path($run_dir);
> +    die "unable to create directory $run_dir\n" if !-d $run_dir;

this is used as part of restoring - what if I restore the same archive
in parallel into two different VMIDs?

> +
> +    return $run_dir;
> +}
> +
> +my sub log_info {
> +    my ($self, $message) = @_;
> +
> +    $self->{'log-function'}->('info', $message);
> +}
> +
> +my sub log_warning {
> +    my ($self, $message) = @_;
> +
> +    $self->{'log-function'}->('warn', $message);
> +}
> +
> +my sub log_error {
> +    my ($self, $message) = @_;
> +
> +    $self->{'log-function'}->('err', $message);
> +}
> +
> +my sub file_contents_from_archive {
> +    my ($self, $archive, $file) = @_;
> +
> +    my $run_dir = prepare_run_dir($archive, "file-contents");
> +
> +    my $raw;
> +
> +    eval {
> +	local $CWD = $run_dir;
> +
> +	$self->{'storage-plugin'}->borg_cmd_extract(
> +	    $self->{scfg},
> +	    $self->{storeid},
> +	    $archive,
> +	    [$file],
> +	);

borg extract has `--stdout`, which would save writing to the FS here
(since this is only used to extract config file, it should be okay)?

> +
> +	$raw = PVE::Tools::file_get_contents("${run_dir}/${file}");
> +    };
> +    my $err = $@;
> +    eval { _remove_tree($run_dir); };
> +    log_warning($self, $@) if $@;
> +    die $err if $err;
> +
> +    return $raw;
> +}
> +
> +# Plugin implementation
> +
> +sub new {
> +    my ($class, $storage_plugin, $scfg, $storeid, $log_function) = @_;
> +
> +    my $self = bless {
> +	scfg => $scfg,
> +	storeid => $storeid,
> +	'storage-plugin' => $storage_plugin,
> +	'log-function' => $log_function,
> +    }, $class;
> +
> +    return $self;
> +}
> +
> +sub provider_name {
> +    my ($self) = @_;
> +
> +    return "Borg";
> +}
> +
> +sub job_hook {
> +    my ($self, $phase, $info) = @_;
> +
> +    if ($phase eq 'start') {
> +	$self->{'job-id'} = $info->{'start-time'};
> +	$self->{password} = $self->{'storage-plugin'}->borg_get_password(
> +	    $self->{scfg}, $self->{storeid});
> +	$self->{'ssh-key-fh'} = $self->{'storage-plugin'}->borg_open_ssh_key(
> +	    $self->{scfg}, $self->{storeid});
> +    } else {
> +	delete $self->{password};

why do we delete this, but don't close the ssh-key-fh ?

> +    }
> +
> +    return;
> +}
> +
> +sub backup_hook {
> +    my ($self, $phase, $vmid, $vmtype, $info) = @_;
> +
> +    if ($phase eq 'start') {
> +	$self->{$vmid}->{'task-size'} = 0;
> +    } elsif ($phase eq 'prepare') {
> +	if ($vmtype eq 'lxc') {
> +	    my $archive = $self->{$vmid}->{archive};
> +	    my $run_dir = prepare_run_dir($archive, "backup-container");
> +	    $self->{$vmid}->{'run-dir'} = $run_dir;
> +
> +	    my $create_dir = sub {
> +		my $dir = shift;
> +		make_path($dir);
> +		die "unable to create directory $dir\n" if !-d $dir;
> +		chown($info->{'backup-user-id'}, -1, $dir)
> +		    or die "unable to change owner for $dir\n";
> +	    };
> +
> +	    $create_dir->("${run_dir}/backup/");
> +	    $create_dir->("${run_dir}/backup/filesystem");
> +	    $create_dir->("${run_dir}/ssh");
> +	    $create_dir->("${run_dir}/.config");
> +	    $create_dir->("${run_dir}/.cache");

so this is a bit tricky.. we need unpriv access (to do the backup), but
we store sensitive things here that we don't actually want to hand out
to everyone..

> +
> +	    for my $subdir ($info->{sources}->@*) {
> +		PVE::Tools::run_command([
> +		    'mount',
> +		    '-o', 'bind,ro',
> +		    "$info->{directory}/${subdir}",
> +		    "${run_dir}/backup/filesystem/${subdir}",
> +		]);
> +	    }
> +	}
> +    } elsif ($phase eq 'end' || $phase eq 'abort') {
> +	if ($vmtype eq 'lxc') {
> +	    my $run_dir = $self->{$vmid}->{'run-dir'};
> +	    eval {
> +		eval { PVE::Tools::run_command(['umount', "${run_dir}/ssh"]); };

this might warrant a comment ;) a tmpfs is mounted there in
backup_container..

> +		eval { PVE::Tools::run_command(['umount', '-R', "${run_dir}/backup/filesystem"]); };
> +		_remove_tree($run_dir);
> +	    };
> +	    die "unable to clean up $run_dir - $@" if $@;
> +	}
> +    }
> +
> +    return;
> +}
> +
> +sub backup_get_mechanism {
> +    my ($self, $vmid, $vmtype) = @_;
> +
> +    return ('block-device', undef) if $vmtype eq 'qemu';
> +    return ('directory', undef) if $vmtype eq 'lxc';
> +
> +    die "unsupported VM type '$vmtype'\n";
> +}
> +
> +sub backup_get_archive_name {
> +    my ($self, $vmid, $vmtype, $backup_time) = @_;
> +
> +    return $self->{$vmid}->{archive} = archive_name($vmtype, $vmid, $backup_time);
> +}
> +
> +sub backup_get_task_size {
> +    my ($self, $vmid) = @_;
> +
> +    return $self->{$vmid}->{'task-size'};
> +}
> +
> +sub backup_handle_log_file {
> +    my ($self, $vmid, $filename) = @_;
> +
> +    return; # don't upload, Proxmox VE keeps the task log too
> +}
> +
> +sub backup_vm {
> +    my ($self, $vmid, $guest_config, $volumes, $info) = @_;
> +
> +    # TODO honor bandwith limit
> +    # TODO discard?
> +
> +    my $archive = $self->{$vmid}->{archive};
> +
> +    my $run_dir = prepare_run_dir($archive, "backup-vm");
> +    my $volume_dir = "${run_dir}/volumes";
> +    make_path($volume_dir);
> +    die "unable to create directory $volume_dir\n" if !-d $volume_dir;
> +
> +    PVE::Tools::file_set_contents("${run_dir}/guest.config", $guest_config);

same here

> +    my $paths = ['./guest.config'];
> +
> +    if (my $firewall_config = $info->{'firewall-config'}) {
> +	PVE::Tools::file_set_contents("${run_dir}/firewall.config", $firewall_config);

and here - these paths are world-readable by default..

> +	push $paths->@*, './firewall.config';
> +    }
> +
> +    for my $devicename (sort keys $volumes->%*) {
> +	my $path = $volumes->{$devicename}->{path};
> +	my $link_name = "${volume_dir}/${devicename}.raw";
> +	symlink($path, $link_name) or die "could not create symlink $link_name -> $path\n";
> +	push $paths->@*, "./volumes/" . basename($link_name, ());
> +    }
> +
> +    # TODO --stats for size?
> +
> +    eval {
> +	local $CWD = $run_dir;
> +
> +	$self->{'storage-plugin'}->borg_cmd_create(
> +	    $self->{scfg},
> +	    $self->{storeid},
> +	    $self->{$vmid}->{archive},
> +	    $paths,
> +	    ['--read-special', '--progress'],
> +	);
> +    };
> +    my $err = $@;
> +    eval { _remove_tree($run_dir) };
> +    log_warning($self, $@) if $@;
> +    die $err if $err;
> +}
> +
> +sub backup_container {
> +    my ($self, $vmid, $guest_config, $exclude_patterns, $info) = @_;
> +
> +    # TODO honor bandwith limit
> +
> +    my $run_dir = $self->{$vmid}->{'run-dir'};
> +    my $backup_dir = "${run_dir}/backup";
> +
> +    my $archive = $self->{$vmid}->{archive};
> +
> +    PVE::Tools::run_command(['mount', '-t', 'tmpfs', '-o', 'size=1M', 'tmpfs', "${run_dir}/ssh"]);
> +
> +    if ($self->{'ssh-key-fh'}) {
> +	my $ssh_key =
> +	    PVE::Tools::safe_read_from($self->{'ssh-key-fh'}, 1024 * 1024, 0, "SSH key file");
> +	PVE::Tools::file_set_contents("${run_dir}/ssh/ssh.key", $ssh_key, 0600);

okay, so this should be fine..

> +    }
> +
> +    if (my $ssh_fingerprint = $self->{scfg}->{'ssh-fingerprint'}) {
> +	my ($server, $port) = $self->{scfg}->@{qw(server port)};
> +	$server = "[$server]" if Net::IP::ip_is_ipv6($server);
> +	$server = "${server}:${port}" if $port;
> +	my $fp_line = "$server $ssh_fingerprint\n";
> +	PVE::Tools::file_set_contents("${run_dir}/ssh/known_hosts", $fp_line, 0600);
> +    }
> +
> +    PVE::Tools::file_set_contents("${backup_dir}/guest.config", $guest_config);

but this

> +    my $paths = ['./guest.config'];
> +
> +    if (my $firewall_config = $info->{'firewall-config'}) {
> +	PVE::Tools::file_set_contents("${backup_dir}/firewall.config", $firewall_config);

and this should also be 0600? or we could chmod the dirs themselves when
creating, to avoid missing paths?

> +	push $paths->@*, './firewall.config';
> +    }
> +
> +    push $paths->@*, "./filesystem";
> +
> +    my $opts = ['--numeric-ids', '--sparse', '--progress'];
> +
> +    for my $pattern ($exclude_patterns->@*) {
> +	if ($pattern =~ m|^/|) {
> +	    push $opts->@*, '-e', "filesystem${pattern}";
> +	} else {
> +	    push $opts->@*, '-e', "filesystem/**${pattern}";
> +	}
> +    }
> +
> +    push $opts->@*, '-e', "filesystem/**lost+found" if $info->{'backup-user-id'} != 0;
> +
> +    # TODO --stats for size?
> +
> +    # Don't make it local to avoid permission denied error when changing back, because the method is
> +    # executed in a user namespace.
> +    $CWD = $backup_dir if $info->{'backup-user-id'} != 0;
> +    {
> +	local $CWD = $backup_dir;
> +	local $ENV{BORG_BASE_DIR} = ${run_dir};
> +	local $ENV{BORG_PASSPHRASE} = $self->{password};
> +
> +	local $ENV{BORG_RSH} =
> +	    "ssh -o \"UserKnownHostsFile ${run_dir}/ssh/known_hosts\" -i ${run_dir}/ssh/ssh.key";
> +
> +	$self->{'storage-plugin'}->borg_cmd_create(
> +	    $self->{scfg},
> +	    $self->{storeid},
> +	    $self->{$vmid}->{archive},
> +	    $paths,
> +	    $opts,
> +	);
> +    }
> +}
> +
> +sub restore_get_mechanism {
> +    my ($self, $volname, $storeid) = @_;
> +
> +    my (undef, $archive) = $self->{'storage-plugin'}->parse_volname($volname);
> +    my ($vmtype) = $archive =~ m!^pve-([^\s-]+)!
> +	or die "cannot parse guest type from archive name '$archive'\n";
> +
> +    return ('qemu-img', $vmtype) if $vmtype eq 'qemu';
> +    return ('directory', $vmtype) if $vmtype eq 'lxc';
> +
> +    die "unexpected guest type '$vmtype'\n";
> +}
> +
> +sub restore_get_guest_config {
> +    my ($self, $volname, $storeid) = @_;
> +
> +    my (undef, $archive) = $self->{'storage-plugin'}->parse_volname($volname);
> +    return file_contents_from_archive($self, $archive, 'guest.config');
> +}
> +
> +sub restore_get_firewall_config {
> +    my ($self, $volname, $storeid) = @_;
> +
> +    my (undef, $archive) = $self->{'storage-plugin'}->parse_volname($volname);
> +    my $config = eval {
> +	file_contents_from_archive($self, $archive, 'firewall.config');
> +    };
> +    if (my $err = $@) {
> +	return if $err =~ m!Include pattern 'firewall\.config' never matched\.!;
> +	die $err;
> +    }
> +    return $config;
> +}
> +
> +sub restore_vm_init {
> +    my ($self, $volname, $storeid) = @_;
> +
> +    my $res = {};
> +
> +    my (undef, $archive, $vmid) = $self->{'storage-plugin'}->parse_volname($volname);
> +    my $mount_point = prepare_run_dir($archive, "restore-vm");
> +
> +    $self->{'storage-plugin'}->borg_cmd_mount(
> +	$self->{scfg},
> +	$self->{storeid},
> +	$archive,
> +	$mount_point,
> +    );

haven't actually tested this code, but what are the permissions like for
this mounted backup archive contents? we don't want to expose guest
volumes as world-readable either..

> +
> +    my @backup_files = glob("$mount_point/volumes/*");
> +    for my $backup_file (@backup_files) {
> +	next if $backup_file !~ m!^(.*/(.*)\.raw)$!; # untaint
> +	($backup_file, my $devicename) = ($1, $2);
> +	# TODO avoid dependency on base plugin?
> +	$res->{$devicename}->{size} = PVE::Storage::Plugin::file_size_info($backup_file);
> +    }
> +
> +    $self->{$volname}->{'mount-point'} = $mount_point;
> +
> +    return $res;
> +}
> +
> +sub restore_vm_cleanup {
> +    my ($self, $volname, $storeid) = @_;
> +
> +    my $mount_point = $self->{$volname}->{'mount-point'} or return;
> +
> +    PVE::Tools::run_command(['umount', $mount_point]);
> +
> +    return;
> +}
> +
> +sub restore_vm_volume_init {
> +    my ($self, $volname, $storeid, $devicename, $info) = @_;
> +
> +    my $mount_point = $self->{$volname}->{'mount-point'}
> +	or die "expected mount point for archive not present\n";
> +
> +    return { 'qemu-img-path' => "${mount_point}/volumes/${devicename}.raw" };
> +}
> +
> +sub restore_vm_volume_cleanup {
> +    my ($self, $volname, $storeid, $devicename, $info) = @_;
> +
> +    return;
> +}
> +
> +sub restore_container_init {
> +    my ($self, $volname, $storeid, $info) = @_;
> +
> +    my (undef, $archive, $vmid) = $self->{'storage-plugin'}->parse_volname($volname);
> +    my $mount_point = prepare_run_dir($archive, "restore-container");
> +
> +    $self->{'storage-plugin'}->borg_cmd_mount(
> +	$self->{scfg},
> +	$self->{storeid},
> +	$archive,
> +	$mount_point,
> +    );

same question here..

> +
> +    $self->{$volname}->{'mount-point'} = $mount_point;
> +
> +    return { 'archive-directory' => "${mount_point}/filesystem" };
> +}
> +
> +sub restore_container_cleanup {
> +    my ($self, $volname, $storeid, $info) = @_;
> +
> +    my $mount_point = $self->{$volname}->{'mount-point'} or return;
> +
> +    PVE::Tools::run_command(['umount', $mount_point]);
> +
> +    return;
> +}
> +
> +1;
> diff --git a/src/PVE/BackupProvider/Plugin/Makefile b/src/PVE/BackupProvider/Plugin/Makefile
> index bedc26e..db08c2d 100644
> --- a/src/PVE/BackupProvider/Plugin/Makefile
> +++ b/src/PVE/BackupProvider/Plugin/Makefile
> @@ -1,4 +1,4 @@
> -SOURCES = Base.pm DirectoryExample.pm
> +SOURCES = Base.pm Borg.pm DirectoryExample.pm
>  
>  .PHONY: install
>  install:
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index 9f9a86b..f4bfc55 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -40,6 +40,7 @@ use PVE::Storage::ZFSPlugin;
>  use PVE::Storage::PBSPlugin;
>  use PVE::Storage::BTRFSPlugin;
>  use PVE::Storage::ESXiPlugin;
> +use PVE::Storage::BorgBackupPlugin;
>  
>  # Storage API version. Increment it on changes in storage API interface.
>  use constant APIVER => 11;
> @@ -66,6 +67,7 @@ PVE::Storage::ZFSPlugin->register();
>  PVE::Storage::PBSPlugin->register();
>  PVE::Storage::BTRFSPlugin->register();
>  PVE::Storage::ESXiPlugin->register();
> +PVE::Storage::BorgBackupPlugin->register();
>  
>  # load third-party plugins
>  if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) {
> diff --git a/src/PVE/Storage/BorgBackupPlugin.pm b/src/PVE/Storage/BorgBackupPlugin.pm
> new file mode 100644
> index 0000000..8f0e721
> --- /dev/null
> +++ b/src/PVE/Storage/BorgBackupPlugin.pm
> @@ -0,0 +1,595 @@
> +package PVE::Storage::BorgBackupPlugin;
> +
> +use strict;
> +use warnings;
> +
> +use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC);
> +use JSON qw(from_json);
> +use MIME::Base64 qw(decode_base64);
> +use Net::IP;
> +use POSIX;
> +
> +use PVE::BackupProvider::Plugin::Borg;
> +use PVE::Tools;
> +
> +use base qw(PVE::Storage::Plugin);
> +
> +my sub borg_repository_uri {
> +    my ($scfg, $storeid) = @_;
> +
> +    my $uri = '';
> +    my $server = $scfg->{server} or die "no server configured for $storeid\n";
> +    my $username = $scfg->{username} or die "no username configured for $storeid\n";
> +    my $prefix = "ssh://$username@";
> +    $server = "[$server]" if Net::IP::ip_is_ipv6($server);
> +    if (my $port = $scfg->{port}) {
> +	$uri = "${prefix}${server}:${port}";
> +    } else {
> +	$uri = "${prefix}${server}";
> +    }
> +    $uri .= $scfg->{'repository-path'};
> +
> +    return $uri;
> +}
> +
> +my sub borg_password_file_name {
> +    my ($scfg, $storeid) = @_;
> +
> +    return "/etc/pve/priv/storage/${storeid}.pw";
> +}
> +
> +my sub borg_set_password {
> +    my ($scfg, $storeid, $password) = @_;
> +
> +    my $pwfile = borg_password_file_name($scfg, $storeid);
> +    mkdir "/etc/pve/priv/storage";
> +
> +    PVE::Tools::file_set_contents($pwfile, "$password\n");
> +}
> +
> +my sub borg_delete_password {
> +    my ($scfg, $storeid) = @_;
> +
> +    my $pwfile = borg_password_file_name($scfg, $storeid);
> +
> +    unlink $pwfile;
> +}
> +
> +sub borg_get_password {
> +    my ($class, $scfg, $storeid) = @_;
> +
> +    my $pwfile = borg_password_file_name($scfg, $storeid);
> +
> +    return PVE::Tools::file_read_firstline($pwfile);
> +}
> +
> +sub borg_cmd_list {
> +    my ($class, $scfg, $storeid) = @_;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +
> +    local $ENV{BORG_PASSPHRASE} = $class->borg_get_password($scfg, $storeid)
> +	if !$ENV{BORG_PASSPHRASE};
> +
> +    my $json = '';
> +    my $cmd = ['borg', 'list', '--json', $uri];
> +
> +    my $errfunc = sub { warn $_[0]; };
> +    my $outfunc = sub { $json .= $_[0]; };
> +
> +    PVE::Tools::run_command(
> +	$cmd, errmsg => "command @$cmd failed", outfunc => $outfunc, errfunc => $errfunc);
> +
> +    my $res = eval { from_json($json) };
> +    die "unable to parse 'borg list' output - $@\n" if $@;
> +    return $res;
> +}
> +
> +sub borg_cmd_create {
> +    my ($class, $scfg, $storeid, $archive, $paths, $opts) = @_;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +
> +    local $ENV{BORG_PASSPHRASE} = $class->borg_get_password($scfg, $storeid)
> +	if !$ENV{BORG_PASSPHRASE};
> +
> +    my $cmd = ['borg', 'create', $opts->@*, "${uri}::${archive}", $paths->@*];
> +
> +    PVE::Tools::run_command($cmd, errmsg => "command @$cmd failed");
> +
> +    return;
> +}
> +
> +sub borg_cmd_extract {
> +    my ($class, $scfg, $storeid, $archive, $paths) = @_;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +
> +    local $ENV{BORG_PASSPHRASE} = $class->borg_get_password($scfg, $storeid)
> +	if !$ENV{BORG_PASSPHRASE};
> +
> +    my $cmd = ['borg', 'extract', "${uri}::${archive}", $paths->@*];
> +
> +    PVE::Tools::run_command($cmd, errmsg => "command @$cmd failed");
> +
> +    return;
> +}
> +
> +sub borg_cmd_delete {
> +    my ($class, $scfg, $storeid, $archive) = @_;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +
> +    local $ENV{BORG_PASSPHRASE} = $class->borg_get_password($scfg, $storeid)
> +	if !$ENV{BORG_PASSPHRASE};
> +
> +    my $cmd = ['borg', 'delete', "${uri}::${archive}"];
> +
> +    PVE::Tools::run_command($cmd, errmsg => "command @$cmd failed");
> +
> +    return;
> +}
> +
> +sub borg_cmd_info {
> +    my ($class, $scfg, $storeid, $archive, $timeout) = @_;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +
> +    local $ENV{BORG_PASSPHRASE} = $class->borg_get_password($scfg, $storeid)
> +	if !$ENV{BORG_PASSPHRASE};
> +
> +    my $json = '';
> +    my $cmd = ['borg', 'info', '--json', "${uri}::${archive}"];
> +
> +    my $errfunc = sub { warn $_[0]; };
> +    my $outfunc = sub { $json .= $_[0]; };
> +
> +    PVE::Tools::run_command(
> +	$cmd,
> +	errmsg => "command @$cmd failed",
> +	timeout => $timeout,
> +	outfunc => $outfunc,
> +	errfunc => $errfunc,
> +    );
> +
> +    my $res = eval { from_json($json) };
> +    die "unable to parse 'borg info' output for archive '$archive' - $@\n" if $@;
> +    return $res;
> +}
> +
> +sub borg_cmd_mount {
> +    my ($class, $scfg, $storeid, $archive, $mount_point) = @_;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +
> +    local $ENV{BORG_PASSPHRASE} = $class->borg_get_password($scfg, $storeid)
> +	if !$ENV{BORG_PASSPHRASE};
> +
> +    my $cmd = ['borg', 'mount', "${uri}::${archive}", $mount_point];
> +
> +    PVE::Tools::run_command($cmd, errmsg => "command @$cmd failed");
> +
> +    return;
> +}
> +
> +my sub parse_backup_time {
> +    my ($time_string) = @_;
> +
> +    my @tm = (POSIX::strptime($time_string, "%FT%TZ"));
> +    # expect sec, min, hour, mday, mon, year
> +    if (grep { !defined($_) } @tm[0..5]) {
> +	warn "error parsing time from string '$time_string'\n";
> +	return 0;
> +    } else {
> +	local $ENV{TZ} = 'UTC'; # time string is UTC
> +
> +	# Fill in isdst to avoid undef warning. No daylight saving time for UTC.
> +	$tm[8] //= 0;
> +
> +	if (my $since_epoch = mktime(@tm)) {
> +	    return int($since_epoch);
> +	} else {
> +	    warn "error parsing time from string '$time_string'\n";
> +	    return 0;
> +	}
> +    }
> +}
> +
> +# Helpers
> +
> +sub type {
> +    return 'borg';
> +}
> +
> +sub plugindata {
> +    return {
> +	content => [ { backup => 1, none => 1 }, { backup => 1 } ],
> +	features => { 'backup-provider' => 1 },
> +    };
> +}
> +
> +sub properties {
> +    return {
> +	'repository-path' => {
> +	    description => "Path to the backup repository",
> +	    type => 'string',
> +	},
> +	'ssh-key' => {
> +	    description => "FIXME", # FIXME
> +	    type => 'string',
> +	},
> +	'ssh-fingerprint' => {
> +	    description => "FIXME", # FIXME
> +	    type => 'string',
> +	},

these should probably get descriptions and formats, but this is titled
WIP :)

> +    };
> +}
> +
> +sub options {
> +    return {
> +	'repository-path' => { fixed => 1 },
> +	server => { fixed => 1 },
> +	port => { optional => 1 },
> +	username => { fixed => 1 },
> +	'ssh-key' => { optional => 1 },
> +	'ssh-fingerprint' => { optional => 1 },
> +	password => { optional => 1 },
> +	disable => { optional => 1 },
> +	nodes => { optional => 1 },
> +	'prune-backups' => { optional => 1 },
> +	'max-protected-backups' => { optional => 1 },
> +    };
> +}
> +
> +sub borg_ssh_key_file_name {
> +    my ($scfg, $storeid) = @_;
> +
> +    return "/etc/pve/priv/storage/${storeid}.ssh.key";
> +}
> +
> +sub borg_set_ssh_key {
> +    my ($scfg, $storeid, $key) = @_;
> +
> +    my $pwfile = borg_ssh_key_file_name($scfg, $storeid);

nit: variable name

> +    mkdir "/etc/pve/priv/storage";
> +
> +    PVE::Tools::file_set_contents($pwfile, "$key\n");
> +}
> +
> +sub borg_delete_ssh_key {
> +    my ($scfg, $storeid) = @_;
> +
> +    my $pwfile = borg_ssh_key_file_name($scfg, $storeid);

same

> +
> +    if (!unlink $pwfile) {
> +	return if $! == ENOENT;
> +	die "failed to delete SSH key! $!\n";
> +    }
> +    delete $scfg->{'ssh-key'};
> +}
> +
> +sub borg_get_ssh_key {
> +    my ($scfg, $storeid) = @_;
> +
> +    my $pwfile = borg_ssh_key_file_name($scfg, $storeid);

same

> +
> +    return PVE::Tools::file_get_contents($pwfile);
> +}
> +
> +# Returns a file handle with FD_CLOEXEC disabled if there is an SSH key , or `undef` if there is
> +# not. Dies on error.
> +sub borg_open_ssh_key {
> +    my ($self, $scfg, $storeid) = @_;
> +
> +    my $ssh_key_file = borg_ssh_key_file_name($scfg, $storeid);
> +
> +    my $keyfd;
> +    if (!open($keyfd, '<', $ssh_key_file)) {
> +	if ($! == ENOENT) {
> +	    die "SSH key configured but no key file found!\n" if $scfg->{'ssh-key'};
> +	    return undef;
> +	}
> +	die "failed to open SSH key: $ssh_key_file: $!\n";
> +    }
> +    my $flags = fcntl($keyfd, F_GETFD, 0)
> +	// die "failed to get file descriptor flags for SSH key FD: $!\n";
> +    fcntl($keyfd, F_SETFD, $flags & ~FD_CLOEXEC)
> +	or die "failed to remove FD_CLOEXEC from SSH key file descriptor\n";
> +
> +    return $keyfd;
> +}
> +
> +# Storage implementation
> +
> +sub on_add_hook {
> +    my ($class, $storeid, $scfg, %param) = @_;
> +
> +    if (defined(my $password = $param{password})) {
> +	borg_set_password($scfg, $storeid, $password);
> +    } else {
> +	borg_delete_password($scfg, $storeid);
> +    }
> +
> +    if (defined(my $ssh_key = delete $param{'ssh-key'})) {
> +	my $decoded = decode_base64($ssh_key);
> +	borg_set_ssh_key($scfg, $storeid, $decoded);
> +	$scfg->{'ssh-key'} = 1;
> +    } else {
> +	borg_delete_ssh_key($scfg, $storeid);
> +    }
> +
> +    return;
> +}
> +
> +sub on_update_hook {
> +    my ($class, $storeid, $scfg, %param) = @_;
> +
> +    if (exists($param{password})) {
> +	if (defined($param{password})) {
> +	    borg_set_password($scfg, $storeid, $param{password});
> +	} else {
> +	    borg_delete_password($scfg, $storeid);
> +	}
> +    }
> +
> +    if (exists($param{'ssh-key'})) {
> +	if (defined(my $ssh_key = delete($param{'ssh-key'}))) {
> +	    my $decoded = decode_base64($ssh_key);
> +
> +	    borg_set_ssh_key($scfg, $storeid, $decoded);
> +	    $scfg->{'ssh-key'} = 1;
> +	} else {
> +	    borg_delete_ssh_key($scfg, $storeid);
> +	}
> +    }
> +
> +    return;
> +}
> +
> +sub on_delete_hook {
> +    my ($class, $storeid, $scfg) = @_;
> +
> +    borg_delete_password($scfg, $storeid);
> +    borg_delete_ssh_key($scfg, $storeid);
> +
> +    return;
> +}
> +
> +sub prune_backups {
> +    my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_;
> +
> +    # FIXME - is 'borg prune' compatible with ours?
> +    die "not implemented";
> +}
> +
> +sub parse_volname {
> +    my ($class, $volname) = @_;
> +
> +    if ($volname =~ m!^backup/(.*)$!) {
> +	my $archive = $1;
> +	if ($archive =~ $PVE::BackupProvider::Plugin::Borg::ARCHIVE_RE_3) {
> +	    return ('backup', $archive, $2);
> +	}
> +    }
> +
> +    die "unable to parse Borg volume name '$volname'\n";
> +}
> +
> +sub path {
> +    my ($class, $scfg, $volname, $storeid, $snapname) = @_;
> +
> +    die "volume snapshot is not possible on Borg volume" if $snapname;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +    my (undef, $archive) = $class->parse_volname($volname);
> +
> +    return "${uri}::${archive}";
> +}
> +
> +sub create_base {
> +    my ($class, $storeid, $scfg, $volname) = @_;
> +
> +    die "cannot create base image in Borg storage\n";
> +}
> +
> +sub clone_image {
> +    my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
> +
> +    die "can't clone images in Borg storage\n";
> +}
> +
> +sub alloc_image {
> +    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
> +
> +    die "can't allocate space in Borg storage\n";
> +}
> +
> +sub free_image {
> +    my ($class, $storeid, $scfg, $volname, $isBase) = @_;
> +
> +    my (undef, $archive) = $class->parse_volname($volname);
> +
> +    borg_cmd_delete($class, $scfg, $storeid, $archive);
> +
> +    return;
> +}
> +
> +sub list_images {
> +    my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
> +
> +    return []; # guest images are not supported, only backups
> +}
> +
> +sub list_volumes {
> +    my ($class, $storeid, $scfg, $vmid, $content_types) = @_;
> +
> +    my $res = [];
> +
> +    return $res if !grep { $_ eq 'backup' } $content_types->@*;
> +
> +    my $archives = $class->borg_cmd_list($scfg, $storeid)->{archives}
> +	or die "expected 'archives' key in 'borg list' JSON output missing\n";
> +
> +    for my $info ($archives->@*) {
> +	my $archive = $info->{archive};
> +	my ($vmtype, $backup_vmid, $time_string) =
> +	    $archive =~ $PVE::BackupProvider::Plugin::Borg::ARCHIVE_RE_3 or next;
> +
> +	next if defined($vmid) && $vmid != $backup_vmid;
> +
> +	push $res->@*, {
> +	    volid => "${storeid}:backup/${archive}",
> +	    size => 0, # FIXME how to cheaply get?
> +	    content => 'backup',
> +	    ctime => parse_backup_time($time_string),
> +	    vmid => $backup_vmid,
> +	    format => "borg-archive",
> +	    subtype => $vmtype,
> +	}
> +    }
> +
> +    return $res;
> +}
> +
> +sub status {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +
> +    my $uri = borg_repository_uri($scfg, $storeid);
> +
> +    my $res;
> +
> +    if ($uri =~ m!^ssh://!) {
> +	#FIXME ssh and df on target?

borg targets will often be locked down to only allow executing borg on
the other end though..

I am not sure what makes sense here tbh..

> +	return;
> +    } else { # $uri is a local path
> +	my $timeout = 2;
> +	$res = PVE::Tools::df($uri, $timeout);
> +
> +	return if !$res || !$res->{total};
> +    }
> +
> +
> +    return ($res->{total}, $res->{avail}, $res->{used}, 1);
> +}
> +
> +sub activate_storage {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +
> +    # TODO how to cheaply check? split ssh and non-ssh?
> +
> +    return 1;
> +}
> +
> +sub deactivate_storage {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +
> +    return 1;
> +}
> +
> +sub activate_volume {
> +    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
> +
> +    die "volume snapshot is not possible on Borg volume" if $snapname;
> +
> +    return 1;
> +}
> +
> +sub deactivate_volume {
> +    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
> +
> +    die "volume snapshot is not possible on Borg volume" if $snapname;
> +
> +    return 1;
> +}
> +
> +sub get_volume_attribute {
> +    my ($class, $scfg, $storeid, $volname, $attribute) = @_;
> +
> +    return;
> +}
> +
> +sub update_volume_attribute {
> +    my ($class, $scfg, $storeid, $volname, $attribute, $value) = @_;
> +
> +    # FIXME notes or protected possible?
> +
> +    die "attribute '$attribute' is not supported on Borg volume";
> +}
> +
> +sub volume_size_info {
> +    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
> +
> +    my (undef, $archive) = $class->parse_volname($volname);
> +    my (undef, undef, $time_string) =
> +	$archive =~ $PVE::BackupProvider::Plugin::Borg::ARCHIVE_RE_3;
> +
> +    my $backup_time = 0;
> +    if ($time_string) {
> +	$backup_time = parse_backup_time($time_string)
> +    } else {
> +	warn "could not parse time from archive name '$archive'\n";
> +    }
> +
> +    my $archives = borg_cmd_info($class, $scfg, $storeid, $archive, $timeout)->{archives}
> +	or die "expected 'archives' key in 'borg info' JSON output missing\n";
> +
> +    my $stats = eval { $archives->[0]->{stats} }
> +	or die "expected entry in 'borg info' JSON output missing\n";
> +    my ($size, $used) = $stats->@{qw(original_size deduplicated_size)};
> +
> +    ($size) = ($size =~ /^(\d+)$/); # untaint
> +    die "size '$size' not an integer\n" if !defined($size);
> +    # coerce back from string
> +    $size = int($size);
> +    ($used) = ($used =~ /^(\d+)$/); # untaint
> +    die "used '$used' not an integer\n" if !defined($used);
> +    # coerce back from string
> +    $used = int($used);
> +
> +    return wantarray ? ($size, 'borg-archive', $used, undef, $backup_time) : $size;
> +}
> +
> +sub volume_resize {
> +    my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
> +
> +    die "volume resize is not possible on Borg volume";
> +}
> +
> +sub volume_snapshot {
> +    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +
> +    die "volume snapshot is not possible on Borg volume";
> +}
> +
> +sub volume_snapshot_rollback {
> +    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +
> +    die "volume snapshot rollback is not possible on Borg volume";
> +}
> +
> +sub volume_snapshot_delete {
> +    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +
> +    die "volume snapshot delete is not possible on Borg volume";
> +}
> +
> +sub volume_has_feature {
> +    my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
> +
> +    return 0;
> +}
> +
> +sub rename_volume {
> +    my ($class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname) = @_;
> +
> +    die "volume rename is not implemented in Borg storage plugin\n";
> +}
> +
> +sub new_backup_provider {
> +    my ($class, $scfg, $storeid, $bandwidth_limit, $log_function) = @_;
> +
> +    return PVE::BackupProvider::Plugin::Borg->new(
> +	$class, $scfg, $storeid, $bandwidth_limit, $log_function);
> +}
> +
> +1;
> diff --git a/src/PVE/Storage/Makefile b/src/PVE/Storage/Makefile
> index acd37f4..9fe2c66 100644
> --- a/src/PVE/Storage/Makefile
> +++ b/src/PVE/Storage/Makefile
> @@ -14,6 +14,7 @@ SOURCES= \
>  	PBSPlugin.pm \
>  	BTRFSPlugin.pm \
>  	LvmThinPlugin.pm \
> +	BorgBackupPlugin.pm \

do we want this one here, while the other one is in Custom?

>  	ESXiPlugin.pm
>  
>  .PHONY: install
> -- 
> 2.39.5
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
> 


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel