From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 3F71B1FF15F for ; Mon, 4 Nov 2024 14:06:12 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0E3CACD63; Mon, 4 Nov 2024 14:06:21 +0100 (CET) Date: Mon, 04 Nov 2024 14:06:13 +0100 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion References: <20241030135537.92595-1-f.schauer@proxmox.com> <20241030135537.92595-3-f.schauer@proxmox.com> In-Reply-To: <20241030135537.92595-3-f.schauer@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.16.0 (https://github.com/astroidmail/astroid) Message-Id: <1730724842.agk2is6zq8.astroid@yuna.none> X-SPAM-LEVEL: Spam detection results: 0 AWL 0.047 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH vma-to-pbs v4 2/6] add support for bulk import of a dump directory X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" On October 30, 2024 2:55 pm, Filip Schauer wrote: > When a path to a directory is provided in the vma_file argument, try to > upload all VMA backups in the directory. This also handles compressed > VMA files, notes and logs. If a vmid is specified with --vmid, only the > backups of that particular vmid are uploaded. > > This is intended for use on a dump directory: > > PBS_FINGERPRINT='PBS_FINGERPRINT' vma-to-pbs \ > --repository 'user@realm!token@server:port:datastore' \ > /var/lib/vz/dump > > Signed-off-by: Filip Schauer > --- > Cargo.toml | 3 + > src/main.rs | 161 +++++++++++++++++++++++++++++++++++++++++++++---- > src/vma2pbs.rs | 64 +++++++++++++++++--- > 3 files changed, 209 insertions(+), 19 deletions(-) > > diff --git a/Cargo.toml b/Cargo.toml > index cd13426..5c6a175 100644 > --- a/Cargo.toml > +++ b/Cargo.toml > @@ -7,9 +7,12 @@ edition = "2021" > [dependencies] > anyhow = "1.0" > bincode = "1.3" > +chrono = "0.4" > hyper = "0.14.5" > +itertools = "0.13" not needed, see below > pico-args = "0.5" > md5 = "0.7.0" > +regex = "1.7" > scopeguard = "1.1.0" > serde = "1.0" > serde_json = "1.0" > diff --git a/src/main.rs b/src/main.rs > index 3e25591..4c5135b 100644 > --- a/src/main.rs > +++ b/src/main.rs > @@ -1,26 +1,35 @@ > use std::ffi::OsString; > +use std::fs::read_dir; > +use std::io::{BufRead, BufReader}; > +use std::path::PathBuf; > > use anyhow::{bail, Context, Error}; > +use chrono::NaiveDateTime; > +use itertools::Itertools; > use proxmox_sys::linux::tty; > use proxmox_time::epoch_i64; > +use regex::Regex; > > mod vma; > mod vma2pbs; > -use vma2pbs::{vma2pbs, BackupVmaToPbsArgs, PbsArgs, VmaBackupArgs}; > +use vma2pbs::{vma2pbs, BackupVmaToPbsArgs, Compression, PbsArgs, VmaBackupArgs}; > > const CMD_HELP: &str = "\ > Usage: vma-to-pbs [OPTIONS] --repository --vmid [vma_file] > > Arguments: > - [vma_file] > + [vma_file | dump_directory] > > Options: > --repository > Repository URL > [--ns ] > Namespace > - --vmid > + [--vmid ] > Backup ID > + This is required if a single VMA file is provided. > + If not specified, bulk import all VMA backups in the provided directory. > + If specified with a dump directory, only import backups of the specified vmid. > [--backup-time ] > Backup timestamp > --fingerprint > @@ -41,6 +50,8 @@ Options: > File containing a comment/notes > [--log-file ] > Log file > + -y, --yes > + Automatic yes to prompts > -h, --help > Print help > -V, --version > @@ -52,7 +63,16 @@ fn parse_args() -> Result { > args.remove(0); // remove the executable path. > > let mut first_later_args_index = 0; > - let options = ["-h", "--help", "-c", "--compress", "-e", "--encrypt"]; > + let options = [ > + "-h", > + "--help", > + "-c", > + "--compress", > + "-e", > + "--encrypt", > + "-y", > + "--yes", > + ]; > > for (i, arg) in args.iter().enumerate() { > if let Some(arg) = arg.to_str() { > @@ -87,7 +107,7 @@ fn parse_args() -> Result { > > let pbs_repository = args.value_from_str("--repository")?; > let namespace = args.opt_value_from_str("--ns")?; > - let vmid = args.value_from_str("--vmid")?; > + let vmid = args.opt_value_from_str("--vmid")?; > let backup_time: Option = args.opt_value_from_str("--backup-time")?; > let backup_time = backup_time.unwrap_or_else(epoch_i64); > let fingerprint = args.opt_value_from_str("--fingerprint")?; > @@ -99,6 +119,7 @@ fn parse_args() -> Result { > let key_password_file: Option = args.opt_value_from_str("--key-password-file")?; > let notes_file: Option = args.opt_value_from_str("--notes-file")?; > let log_file_path: Option = args.opt_value_from_str("--log-file")?; > + let yes = args.contains(["-y", "--yes"]); > > match (encrypt, keyfile.is_some()) { > (true, false) => bail!("--encrypt requires a --keyfile!"), > @@ -196,15 +217,131 @@ fn parse_args() -> Result { > encrypt, > }; > > - let vma_args = VmaBackupArgs { > - vma_file_path: vma_file_path.cloned(), > - backup_id: vmid, > - backup_time, > - notes, > - log_file_path, > + let bulk = > + vma_file_path > + .map(PathBuf::from) > + .and_then(|path| if path.is_dir() { Some(path) } else { None }); > + > + let grouped_vmas = if let Some(dump_dir_path) = bulk { grouped_vmas should still be a map, not a vec of vec.. e.g., something like this (requires some more adaptation - while this could use itertools, I don't think it's worth to pull that in if the same can be had with a single fold invocation): @@ -298,12 +298,16 @@ fn parse_args() -> Result { vmas.sort_by_key(|d| d.backup_time); let total_vma_count = vmas.len(); - let mut grouped_vmas: Vec<_> = vmas - .into_iter() - .into_group_map_by(|d| d.backup_id.clone()) - .into_values() - .collect(); - grouped_vmas.sort_by_key(|d| d[0].backup_id.clone()); + let grouped_vmas = vmas.into_iter().fold( + HashMap::new(), + |mut grouped: HashMap>, vma_args| { + grouped + .entry(vma_args.backup_id.clone()) + .or_default() + .push(vma_args); + grouped + }, + ); log::info!( "Found {} backup archive(s) of {} different VMID(s):", @@ -311,12 +315,8 @@ fn parse_args() -> Result { grouped_vmas.len() ); - for vma_group in &grouped_vmas { - log::info!( - "- VMID {}: {} backups", - vma_group[0].backup_id, - vma_group.len() - ); + for (vma_group, vma_args) in &grouped_vmas { + log::info!("- VMID {}: {} backups", vma_group, vma_args.len()); } if !yes { > + let re = Regex::new( > + r"vzdump-qemu-(\d+)-(\d{4}_\d{2}_\d{2}-\d{2}_\d{2}_\d{2}).vma(|.zst|.lzo|.gz)$", > + )?; > + > + let mut vmas = Vec::new(); > + > + for entry in read_dir(dump_dir_path)? { > + let entry = entry?; > + let path = entry.path(); > + > + if !path.is_file() { > + continue; > + } > + > + if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) { > + let Some((_, [backup_id, timestr, ext])) = > + re.captures(file_name).map(|c| c.extract()) > + else { > + // Skip the file, since it is not a VMA backup > + continue; > + }; > + > + if let Some(ref vmid) = vmid { > + if backup_id != vmid { > + // Skip the backup, since it does not match the specified vmid > + continue; > + } > + } > + > + let compression = match ext { > + "" => None, > + ".zst" => Some(Compression::Zstd), > + ".lzo" => Some(Compression::Lzo), > + ".gz" => Some(Compression::GZip), > + _ => bail!("Unexpected file extension: {ext}"), > + }; > + > + let backup_time = NaiveDateTime::parse_from_str(timestr, "%Y_%m_%d-%H_%M_%S")? > + .and_utc() > + .timestamp(); > + > + let notes_path = path.with_file_name(format!("{}.notes", file_name)); > + let notes = proxmox_sys::fs::file_read_optional_string(notes_path)?; > + > + let log_path = path.with_file_name(format!("{}.log", file_name)); > + let log_file_path = if log_path.exists() { > + Some(log_path.to_path_buf().into_os_string()) > + } else { > + None > + }; > + > + let backup_args = VmaBackupArgs { > + vma_file_path: Some(path.clone().into()), > + compression, > + backup_id: backup_id.to_string(), > + backup_time, > + notes, > + log_file_path, > + }; > + vmas.push(backup_args); > + } > + } > + > + vmas.sort_by_key(|d| d.backup_time); > + let total_vma_count = vmas.len(); > + let mut grouped_vmas: Vec<_> = vmas > + .into_iter() > + .into_group_map_by(|d| d.backup_id.clone()) > + .into_values() > + .collect(); > + grouped_vmas.sort_by_key(|d| d[0].backup_id.clone()); > + > + println!( > + "Found {} backup archive(s) of {} different VMID(s):", > + total_vma_count, > + grouped_vmas.len() > + ); if we don't find any, we should print something else here and exit? > + > + for vma_group in &grouped_vmas { > + println!( > + "- VMID {}: {} backups", > + vma_group[0].backup_id, > + vma_group.len() > + ); > + } > + > + if !yes { > + loop { > + eprint!("Proceed with the bulk import? (y/n): "); > + let mut line = String::new(); > + > + BufReader::new(std::io::stdin()).read_line(&mut line)?; > + let trimmed = line.trim(); > + if trimmed == "y" || trimmed == "Y" { > + break; > + } else if trimmed == "n" || trimmed == "N" { > + bail!("Bulk import was not confirmed."); > + } this maybe should mimic what we do in proxmox_router when prompting for confirmation? e.g., flush stdout, have a default value, ..? should we abort after a few loops? > + } > + } > + > + grouped_vmas _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel