From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 4AA151FF15C for ; Wed, 30 Oct 2024 14:55:48 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 542FE1A469; Wed, 30 Oct 2024 14:55:51 +0100 (CET) From: Filip Schauer To: pbs-devel@lists.proxmox.com Date: Wed, 30 Oct 2024 14:55:33 +0100 Message-Id: <20241030135537.92595-3-f.schauer@proxmox.com> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20241030135537.92595-1-f.schauer@proxmox.com> References: <20241030135537.92595-1-f.schauer@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.033 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH vma-to-pbs v4 2/6] add support for bulk import of a dump directory X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" When a path to a directory is provided in the vma_file argument, try to upload all VMA backups in the directory. This also handles compressed VMA files, notes and logs. If a vmid is specified with --vmid, only the backups of that particular vmid are uploaded. This is intended for use on a dump directory: PBS_FINGERPRINT='PBS_FINGERPRINT' vma-to-pbs \ --repository 'user@realm!token@server:port:datastore' \ /var/lib/vz/dump Signed-off-by: Filip Schauer --- Cargo.toml | 3 + src/main.rs | 161 +++++++++++++++++++++++++++++++++++++++++++++---- src/vma2pbs.rs | 64 +++++++++++++++++--- 3 files changed, 209 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cd13426..5c6a175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,12 @@ edition = "2021" [dependencies] anyhow = "1.0" bincode = "1.3" +chrono = "0.4" hyper = "0.14.5" +itertools = "0.13" pico-args = "0.5" md5 = "0.7.0" +regex = "1.7" scopeguard = "1.1.0" serde = "1.0" serde_json = "1.0" diff --git a/src/main.rs b/src/main.rs index 3e25591..4c5135b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,26 +1,35 @@ use std::ffi::OsString; +use std::fs::read_dir; +use std::io::{BufRead, BufReader}; +use std::path::PathBuf; use anyhow::{bail, Context, Error}; +use chrono::NaiveDateTime; +use itertools::Itertools; use proxmox_sys::linux::tty; use proxmox_time::epoch_i64; +use regex::Regex; mod vma; mod vma2pbs; -use vma2pbs::{vma2pbs, BackupVmaToPbsArgs, PbsArgs, VmaBackupArgs}; +use vma2pbs::{vma2pbs, BackupVmaToPbsArgs, Compression, PbsArgs, VmaBackupArgs}; const CMD_HELP: &str = "\ Usage: vma-to-pbs [OPTIONS] --repository --vmid [vma_file] Arguments: - [vma_file] + [vma_file | dump_directory] Options: --repository Repository URL [--ns ] Namespace - --vmid + [--vmid ] Backup ID + This is required if a single VMA file is provided. + If not specified, bulk import all VMA backups in the provided directory. + If specified with a dump directory, only import backups of the specified vmid. [--backup-time ] Backup timestamp --fingerprint @@ -41,6 +50,8 @@ Options: File containing a comment/notes [--log-file ] Log file + -y, --yes + Automatic yes to prompts -h, --help Print help -V, --version @@ -52,7 +63,16 @@ fn parse_args() -> Result { args.remove(0); // remove the executable path. let mut first_later_args_index = 0; - let options = ["-h", "--help", "-c", "--compress", "-e", "--encrypt"]; + let options = [ + "-h", + "--help", + "-c", + "--compress", + "-e", + "--encrypt", + "-y", + "--yes", + ]; for (i, arg) in args.iter().enumerate() { if let Some(arg) = arg.to_str() { @@ -87,7 +107,7 @@ fn parse_args() -> Result { let pbs_repository = args.value_from_str("--repository")?; let namespace = args.opt_value_from_str("--ns")?; - let vmid = args.value_from_str("--vmid")?; + let vmid = args.opt_value_from_str("--vmid")?; let backup_time: Option = args.opt_value_from_str("--backup-time")?; let backup_time = backup_time.unwrap_or_else(epoch_i64); let fingerprint = args.opt_value_from_str("--fingerprint")?; @@ -99,6 +119,7 @@ fn parse_args() -> Result { let key_password_file: Option = args.opt_value_from_str("--key-password-file")?; let notes_file: Option = args.opt_value_from_str("--notes-file")?; let log_file_path: Option = args.opt_value_from_str("--log-file")?; + let yes = args.contains(["-y", "--yes"]); match (encrypt, keyfile.is_some()) { (true, false) => bail!("--encrypt requires a --keyfile!"), @@ -196,15 +217,131 @@ fn parse_args() -> Result { encrypt, }; - let vma_args = VmaBackupArgs { - vma_file_path: vma_file_path.cloned(), - backup_id: vmid, - backup_time, - notes, - log_file_path, + let bulk = + vma_file_path + .map(PathBuf::from) + .and_then(|path| if path.is_dir() { Some(path) } else { None }); + + let grouped_vmas = if let Some(dump_dir_path) = bulk { + let re = Regex::new( + r"vzdump-qemu-(\d+)-(\d{4}_\d{2}_\d{2}-\d{2}_\d{2}_\d{2}).vma(|.zst|.lzo|.gz)$", + )?; + + let mut vmas = Vec::new(); + + for entry in read_dir(dump_dir_path)? { + let entry = entry?; + let path = entry.path(); + + if !path.is_file() { + continue; + } + + if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) { + let Some((_, [backup_id, timestr, ext])) = + re.captures(file_name).map(|c| c.extract()) + else { + // Skip the file, since it is not a VMA backup + continue; + }; + + if let Some(ref vmid) = vmid { + if backup_id != vmid { + // Skip the backup, since it does not match the specified vmid + continue; + } + } + + let compression = match ext { + "" => None, + ".zst" => Some(Compression::Zstd), + ".lzo" => Some(Compression::Lzo), + ".gz" => Some(Compression::GZip), + _ => bail!("Unexpected file extension: {ext}"), + }; + + let backup_time = NaiveDateTime::parse_from_str(timestr, "%Y_%m_%d-%H_%M_%S")? + .and_utc() + .timestamp(); + + let notes_path = path.with_file_name(format!("{}.notes", file_name)); + let notes = proxmox_sys::fs::file_read_optional_string(notes_path)?; + + let log_path = path.with_file_name(format!("{}.log", file_name)); + let log_file_path = if log_path.exists() { + Some(log_path.to_path_buf().into_os_string()) + } else { + None + }; + + let backup_args = VmaBackupArgs { + vma_file_path: Some(path.clone().into()), + compression, + backup_id: backup_id.to_string(), + backup_time, + notes, + log_file_path, + }; + vmas.push(backup_args); + } + } + + vmas.sort_by_key(|d| d.backup_time); + let total_vma_count = vmas.len(); + let mut grouped_vmas: Vec<_> = vmas + .into_iter() + .into_group_map_by(|d| d.backup_id.clone()) + .into_values() + .collect(); + grouped_vmas.sort_by_key(|d| d[0].backup_id.clone()); + + println!( + "Found {} backup archive(s) of {} different VMID(s):", + total_vma_count, + grouped_vmas.len() + ); + + for vma_group in &grouped_vmas { + println!( + "- VMID {}: {} backups", + vma_group[0].backup_id, + vma_group.len() + ); + } + + if !yes { + loop { + eprint!("Proceed with the bulk import? (y/n): "); + let mut line = String::new(); + + BufReader::new(std::io::stdin()).read_line(&mut line)?; + let trimmed = line.trim(); + if trimmed == "y" || trimmed == "Y" { + break; + } else if trimmed == "n" || trimmed == "N" { + bail!("Bulk import was not confirmed."); + } + } + } + + grouped_vmas + } else if let Some(vmid) = vmid { + vec![vec![VmaBackupArgs { + vma_file_path: vma_file_path.cloned(), + compression: None, + backup_id: vmid, + backup_time, + notes, + log_file_path, + }]] + } else { + bail!("No vmid specified for single backup file"); }; - let options = BackupVmaToPbsArgs { pbs_args, vma_args }; + let options = BackupVmaToPbsArgs { + pbs_args, + grouped_vmas, + }; Ok(options) } diff --git a/src/vma2pbs.rs b/src/vma2pbs.rs index 37ea308..a3f1eee 100644 --- a/src/vma2pbs.rs +++ b/src/vma2pbs.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::ffi::{c_char, CStr, CString, OsString}; use std::fs::File; use std::io::{stdin, BufRead, BufReader, Read}; +use std::process::{Command, Stdio}; use std::ptr; use std::time::SystemTime; @@ -30,7 +31,7 @@ const VMA_CLUSTER_SIZE: usize = 65536; pub struct BackupVmaToPbsArgs { pub pbs_args: PbsArgs, - pub vma_args: VmaBackupArgs, + pub grouped_vmas: Vec>, } pub struct PbsArgs { @@ -45,8 +46,15 @@ pub struct PbsArgs { pub encrypt: bool, } +pub enum Compression { + Zstd, + Lzo, + GZip, +} + pub struct VmaBackupArgs { pub vma_file_path: Option, + pub compression: Option, pub backup_id: String, pub backup_time: i64, pub notes: Option, @@ -468,7 +476,19 @@ pub fn vma2pbs(args: BackupVmaToPbsArgs) -> Result<(), Error> { let start_transfer_time = SystemTime::now(); - upload_vma_file(pbs_args, &args.vma_args)?; + for vma_group in args.grouped_vmas { + for backup_args in vma_group { + if let Err(e) = upload_vma_file(pbs_args, &backup_args) { + eprintln!( + "Failed to upload vma file at {:?} - {}", + backup_args.vma_file_path.unwrap_or("(stdin)".into()), + e + ); + println!("Skipping VMID {}", backup_args.backup_id); + break; + } + } + } let transfer_duration = SystemTime::now().duration_since(start_transfer_time)?; let total_seconds = transfer_duration.as_secs(); @@ -481,13 +501,43 @@ pub fn vma2pbs(args: BackupVmaToPbsArgs) -> Result<(), Error> { } fn upload_vma_file(pbs_args: &PbsArgs, backup_args: &VmaBackupArgs) -> Result<(), Error> { - let vma_file: Box = match &backup_args.vma_file_path { - Some(vma_file_path) => match File::open(vma_file_path) { - Err(why) => return Err(anyhow!("Couldn't open file: {}", why)), - Ok(file) => Box::new(BufReader::new(file)), + match &backup_args.vma_file_path { + Some(vma_file_path) => println!("Uploading VMA backup from {:?}", vma_file_path), + None => println!("Uploading VMA backup from (stdin)"), + }; + + let vma_file: Box = match &backup_args.compression { + Some(compression) => { + let vma_file_path = backup_args + .vma_file_path + .as_ref() + .expect("No VMA file path provided"); + let mut cmd = match compression { + Compression::Zstd => { + let mut cmd = Command::new("zstd"); + cmd.args(["-q", "-d", "-c"]); + cmd + } + Compression::Lzo => { + let mut cmd = Command::new("lzop"); + cmd.args(["-d", "-c"]); + cmd + } + Compression::GZip => Command::new("zcat"), + }; + let process = cmd.arg(vma_file_path).stdout(Stdio::piped()).spawn()?; + let stdout = process.stdout.expect("Failed to capture stdout"); + Box::new(BufReader::new(stdout)) + } + None => match &backup_args.vma_file_path { + Some(vma_file_path) => match File::open(vma_file_path) { + Err(why) => return Err(anyhow!("Couldn't open file: {}", why)), + Ok(file) => Box::new(BufReader::new(file)), + }, + None => Box::new(BufReader::new(stdin())), }, - None => Box::new(BufReader::new(stdin())), }; + let vma_reader = VmaReader::new(vma_file)?; let pbs = create_pbs_backup_task(pbs_args, backup_args)?; -- 2.39.5 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel