From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id BBDF41FF15C for ; Wed, 24 Jul 2024 18:19:13 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E2F551F3DA; Wed, 24 Jul 2024 18:19:12 +0200 (CEST) From: Filip Schauer To: pbs-devel@lists.proxmox.com Date: Wed, 24 Jul 2024 18:18:54 +0200 Message-Id: <20240724161856.398271-3-f.schauer@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240724161856.398271-1-f.schauer@proxmox.com> References: <20240724161856.398271-1-f.schauer@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.057 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH vma-to-pbs 2/4] add support for bulk import of a dump directory X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" When no vmid is provided, treat the vma_file argument as a path to a directory containing VMA backups. This also handles compressed VMA files, notes and logs. This makes it ideal for use on a dump directory: PBS_FINGERPRINT='PBS_FINGERPRINT' vma-to-pbs \ --repository 'user@realm!token@server:port:datastore' \ /var/lib/vz/dump Signed-off-by: Filip Schauer --- Cargo.toml | 3 + src/main.rs | 114 +++++++++++++++++++++--- src/vma2pbs.rs | 234 +++++++++++++++++++++++++++++++------------------ 3 files changed, 254 insertions(+), 97 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0522902..54b5ebb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,13 +7,16 @@ edition = "2021" [dependencies] anyhow = "1.0" bincode = "1.3" +chrono = "0.4" hyper = "0.14.5" pico-args = "0.4" md5 = "0.7.0" +regex = "1.7" scopeguard = "1.1.0" serde = "1.0" serde_json = "1.0" serde-big-array = "0.4.1" +walkdir = "2" proxmox-async = "0.4" proxmox-io = "1.0.1" diff --git a/src/main.rs b/src/main.rs index de789c1..233992b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,26 +1,30 @@ use std::ffi::OsString; +use std::path::PathBuf; use anyhow::{bail, Context, Error}; +use chrono::NaiveDateTime; use proxmox_sys::linux::tty; use proxmox_time::epoch_i64; +use regex::Regex; +use walkdir::WalkDir; mod vma; mod vma2pbs; -use vma2pbs::{backup_vma_to_pbs, BackupVmaToPbsArgs}; +use vma2pbs::{vma2pbs, BackupVmaToPbsArgs, Compression, PbsArgs, VmaBackupArgs}; const CMD_HELP: &str = "\ Usage: vma-to-pbs [OPTIONS] --repository --vmid [vma_file] Arguments: - [vma_file] + [vma_file | dump_directory] Options: --repository Repository URL [--ns ] Namespace - --vmid - Backup ID + [--vmid ] + Backup ID (If not specified, bulk import all VMA backups in the provided directory) [--backup-time ] Backup timestamp --fingerprint @@ -87,7 +91,7 @@ fn parse_args() -> Result { let pbs_repository = args.value_from_str("--repository")?; let namespace = args.opt_value_from_str("--ns")?; - let vmid = args.value_from_str("--vmid")?; + let vmid = args.opt_value_from_str("--vmid")?; let backup_time: Option = args.opt_value_from_str("--backup-time")?; let backup_time = backup_time.unwrap_or_else(epoch_i64); let fingerprint = args.opt_value_from_str("--fingerprint")?; @@ -184,12 +188,9 @@ fn parse_args() -> Result { None }; - let options = BackupVmaToPbsArgs { - vma_file_path: vma_file_path.cloned(), + let pbs_args = PbsArgs { pbs_repository, namespace, - backup_id: vmid, - backup_time, pbs_password, keyfile, key_password, @@ -197,16 +198,105 @@ fn parse_args() -> Result { fingerprint, compress, encrypt, - notes, - log_file_path, }; + let mut vmas = Vec::new(); + + if let Some(vmid) = vmid { + let backup_args = VmaBackupArgs { + vma_file_path: vma_file_path.cloned(), + compression: None, + backup_id: vmid, + backup_time, + notes, + log_file_path, + }; + vmas.push(backup_args); + } else { + let dump_dir_path = + PathBuf::from(vma_file_path.expect("no directory specified for bulk import")); + + if !dump_dir_path.is_dir() { + bail!("specified path for bulk import is not a directory"); + } + + for entry in WalkDir::new(dump_dir_path) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + + if !path.is_file() { + continue; + } + + if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) { + let re = Regex::new( + r"vzdump-qemu-(\d+)-(\d{4}_\d{2}_\d{2}-\d{2}_\d{2}_\d{2}).vma(|.zst|.lzo|.gz)$", + )?; + + let caps = match re.captures(file_name) { + Some(caps) => caps, + None => continue, + }; + + let Some(vmid) = caps.get(1) else { continue }; + let Some(timestr) = caps.get(2) else { continue }; + let Some(ext) = caps.get(3) else { continue }; + + let compression = match ext.as_str() { + "" => None, + ".zst" => Some(Compression::Zstd), + ".lzo" => Some(Compression::Lzo), + ".gz" => Some(Compression::GZip), + _ => continue, + }; + + let backup_time = + NaiveDateTime::parse_from_str(timestr.as_str(), "%Y_%m_%d-%H_%M_%S")? + .timestamp(); + + let mut notes_path_os_string: OsString = path.into(); + notes_path_os_string.push(".notes"); + let notes_path: PathBuf = notes_path_os_string.into(); + let notes = if notes_path.exists() { + Some(std::fs::read_to_string(notes_path)?) + } else { + None + }; + + let mut log_path_os_string: OsString = path.into(); + log_path_os_string.push(".log"); + let log_path: PathBuf = log_path_os_string.into(); + let log_file_path = if log_path.exists() { + Some(log_path.to_path_buf().into_os_string()) + } else { + None + }; + + let backup_args = VmaBackupArgs { + vma_file_path: Some(path.into()), + compression, + backup_id: vmid.as_str().to_string(), + backup_time, + notes, + log_file_path, + }; + vmas.push(backup_args); + } + } + + vmas.sort_by_key(|d| d.backup_time); + } + + let options = BackupVmaToPbsArgs { pbs_args, vmas }; + Ok(options) } fn main() -> Result<(), Error> { let args = parse_args()?; - backup_vma_to_pbs(args)?; + vma2pbs(args)?; Ok(()) } diff --git a/src/vma2pbs.rs b/src/vma2pbs.rs index d2ce437..3e9689d 100644 --- a/src/vma2pbs.rs +++ b/src/vma2pbs.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::ffi::{c_char, CStr, CString, OsString}; use std::fs::File; use std::io::{stdin, BufRead, BufReader, Read}; +use std::process::{Command, Stdio}; use std::ptr; use std::time::SystemTime; @@ -29,11 +30,13 @@ use crate::vma::VmaReader; const VMA_CLUSTER_SIZE: usize = 65536; pub struct BackupVmaToPbsArgs { - pub vma_file_path: Option, + pub pbs_args: PbsArgs, + pub vmas: Vec, +} + +pub struct PbsArgs { pub pbs_repository: String, pub namespace: Option, - pub backup_id: String, - pub backup_time: i64, pub pbs_password: String, pub keyfile: Option, pub key_password: Option, @@ -41,6 +44,19 @@ pub struct BackupVmaToPbsArgs { pub fingerprint: String, pub compress: bool, pub encrypt: bool, +} + +pub enum Compression { + Zstd, + Lzo, + GZip, +} + +pub struct VmaBackupArgs { + pub vma_file_path: Option, + pub compression: Option, + pub backup_id: String, + pub backup_time: i64, pub notes: Option, pub log_file_path: Option, } @@ -61,25 +77,25 @@ fn handle_pbs_error(pbs_err: *mut c_char, function_name: &str) -> Result<(), Err bail!("{function_name} failed: {pbs_err_str}"); } -fn create_pbs_backup_task(args: &BackupVmaToPbsArgs) -> Result<*mut ProxmoxBackupHandle, Error> { - println!("PBS repository: {}", args.pbs_repository); - if let Some(ns) = &args.namespace { +fn create_pbs_backup_task(pbs_args: &PbsArgs, backup_args: &VmaBackupArgs) -> Result<*mut ProxmoxBackupHandle, Error> { + println!("PBS repository: {}", pbs_args.pbs_repository); + if let Some(ns) = &pbs_args.namespace { println!("PBS namespace: {}", ns); } - println!("PBS fingerprint: {}", args.fingerprint); - println!("compress: {}", args.compress); - println!("encrypt: {}", args.encrypt); + println!("PBS fingerprint: {}", pbs_args.fingerprint); + println!("compress: {}", pbs_args.compress); + println!("encrypt: {}", pbs_args.encrypt); - println!("backup time: {}", epoch_to_rfc3339(args.backup_time)?); + println!("backup time: {}", epoch_to_rfc3339(backup_args.backup_time)?); let mut pbs_err: *mut c_char = ptr::null_mut(); - let pbs_repository_cstr = CString::new(args.pbs_repository.as_str())?; - let ns_cstr = CString::new(args.namespace.as_deref().unwrap_or(""))?; - let backup_id_cstr = CString::new(args.backup_id.as_str())?; - let pbs_password_cstr = CString::new(args.pbs_password.as_str())?; - let fingerprint_cstr = CString::new(args.fingerprint.as_str())?; - let keyfile_cstr = args + let pbs_repository_cstr = CString::new(pbs_args.pbs_repository.as_str())?; + let ns_cstr = CString::new(pbs_args.namespace.as_deref().unwrap_or(""))?; + let backup_id_cstr = CString::new(backup_args.backup_id.as_str())?; + let pbs_password_cstr = CString::new(pbs_args.pbs_password.as_str())?; + let fingerprint_cstr = CString::new(pbs_args.fingerprint.as_str())?; + let keyfile_cstr = pbs_args .keyfile .as_ref() .map(|v| CString::new(v.as_str()).unwrap()); @@ -87,7 +103,7 @@ fn create_pbs_backup_task(args: &BackupVmaToPbsArgs) -> Result<*mut ProxmoxBacku .as_ref() .map(|v| v.as_ptr()) .unwrap_or(ptr::null()); - let key_password_cstr = args + let key_password_cstr = pbs_args .key_password .as_ref() .map(|v| CString::new(v.as_str()).unwrap()); @@ -95,7 +111,7 @@ fn create_pbs_backup_task(args: &BackupVmaToPbsArgs) -> Result<*mut ProxmoxBacku .as_ref() .map(|v| v.as_ptr()) .unwrap_or(ptr::null()); - let master_keyfile_cstr = args + let master_keyfile_cstr = pbs_args .master_keyfile .as_ref() .map(|v| CString::new(v.as_str()).unwrap()); @@ -108,14 +124,14 @@ fn create_pbs_backup_task(args: &BackupVmaToPbsArgs) -> Result<*mut ProxmoxBacku pbs_repository_cstr.as_ptr(), ns_cstr.as_ptr(), backup_id_cstr.as_ptr(), - args.backup_time as u64, + backup_args.backup_time as u64, PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE, pbs_password_cstr.as_ptr(), keyfile_ptr, key_password_ptr, master_keyfile_ptr, - args.compress, - args.encrypt, + pbs_args.compress, + pbs_args.encrypt, fingerprint_cstr.as_ptr(), &mut pbs_err, ); @@ -361,17 +377,24 @@ where Ok(()) } -fn pbs_client_setup(args: &BackupVmaToPbsArgs) -> Result<(HttpClient, String, Value), Error> { - let repo: BackupRepository = args.pbs_repository.parse()?; +fn pbs_client_setup( + pbs_args: &PbsArgs, + backup_args: &VmaBackupArgs, +) -> Result<(HttpClient, String, Value), Error> { + let repo: BackupRepository = pbs_args.pbs_repository.parse()?; let options = HttpClientOptions::new_interactive( - Some(args.pbs_password.clone()), - Some(args.fingerprint.clone()), + Some(pbs_args.pbs_password.clone()), + Some(pbs_args.fingerprint.clone()), ); let client = HttpClient::new(repo.host(), repo.port(), repo.auth_id(), options)?; - let backup_dir = BackupDir::from((BackupType::Vm, args.backup_id.clone(), args.backup_time)); + let backup_dir = BackupDir::from(( + BackupType::Vm, + backup_args.backup_id.clone(), + backup_args.backup_time, + )); - let namespace = match &args.namespace { + let namespace = match &pbs_args.namespace { Some(namespace) => BackupNamespace::new(namespace)?, None => BackupNamespace::root(), }; @@ -386,45 +409,44 @@ fn pbs_client_setup(args: &BackupVmaToPbsArgs) -> Result<(HttpClient, String, Va fn upload_log( client: &HttpClient, - args: &BackupVmaToPbsArgs, + log_file_path: &OsString, + pbs_args: &PbsArgs, store: &str, request_args: Value, ) -> Result<(), Error> { - if let Some(log_file_path) = &args.log_file_path { - let path = format!("api2/json/admin/datastore/{}/upload-backup-log", store); - let data = std::fs::read(log_file_path)?; - - let blob = if args.encrypt { - let crypt_config = match &args.keyfile { - None => None, - Some(keyfile) => { - let key = std::fs::read(keyfile)?; - let (key, _created, _) = decrypt_key(&key, &|| -> Result, Error> { - match &args.key_password { - Some(key_password) => Ok(key_password.clone().into_bytes()), - None => bail!("no key password provided"), - } - })?; - let crypt_config = CryptConfig::new(key)?; - Some(crypt_config) - } - }; - - DataBlob::encode(&data, crypt_config.as_ref(), args.compress)? - } else { - // fixme: howto sign log? - DataBlob::encode(&data, None, args.compress)? + let path = format!("api2/json/admin/datastore/{}/upload-backup-log", store); + let data = std::fs::read(log_file_path)?; + + let blob = if pbs_args.encrypt { + let crypt_config = match &pbs_args.keyfile { + None => None, + Some(keyfile) => { + let key = std::fs::read(keyfile)?; + let (key, _created, _) = decrypt_key(&key, &|| -> Result, Error> { + match &pbs_args.key_password { + Some(key_password) => Ok(key_password.clone().into_bytes()), + None => bail!("no key password provided"), + } + })?; + let crypt_config = CryptConfig::new(key)?; + Some(crypt_config) + } }; - let body = hyper::Body::from(blob.into_inner()); + DataBlob::encode(&data, crypt_config.as_ref(), pbs_args.compress)? + } else { + // fixme: howto sign log? + DataBlob::encode(&data, None, pbs_args.compress)? + }; - block_on(async { - client - .upload("application/octet-stream", body, &path, Some(request_args)) - .await - .unwrap(); - }); - } + let body = hyper::Body::from(blob.into_inner()); + + block_on(async { + client + .upload("application/octet-stream", body, &path, Some(request_args)) + .await + .unwrap(); + }); Ok(()) } @@ -444,17 +466,64 @@ fn set_notes( Ok(()) } -pub fn backup_vma_to_pbs(args: BackupVmaToPbsArgs) -> Result<(), Error> { - let vma_file: Box = match &args.vma_file_path { - Some(vma_file_path) => match File::open(vma_file_path) { - Err(why) => return Err(anyhow!("Couldn't open file: {}", why)), - Ok(file) => Box::new(BufReader::new(file)), +pub fn vma2pbs(args: BackupVmaToPbsArgs) -> Result<(), Error> { + let start_transfer_time = SystemTime::now(); + + for backup_args in args.vmas { + upload_vma_file(&args.pbs_args, backup_args)?; + } + + let transfer_duration = SystemTime::now().duration_since(start_transfer_time)?; + let total_seconds = transfer_duration.as_secs(); + let minutes = total_seconds / 60; + let seconds = total_seconds % 60; + let milliseconds = transfer_duration.as_millis() % 1000; + println!("Backup finished within {minutes} minutes, {seconds} seconds and {milliseconds} ms"); + + Ok(()) +} + +fn upload_vma_file(pbs_args: &PbsArgs, backup_args: VmaBackupArgs) -> Result<(), Error> { + match &backup_args.vma_file_path { + Some(vma_file_path) => println!("Uploading VMA backup from {:?}", vma_file_path), + None => println!("Uploading VMA backup from (stdin)"), + }; + + let vma_file: Box = match &backup_args.compression { + Some(compression) => { + let vma_file_path = backup_args + .vma_file_path + .as_ref() + .expect("No VMA file path provided"); + let mut cmd = match compression { + Compression::Zstd => { + let mut cmd = Command::new("zstd"); + cmd.args(["-q", "-d", "-c"]); + cmd + } + Compression::Lzo => { + let mut cmd = Command::new("lzop"); + cmd.args(["-d", "-c"]); + cmd + } + Compression::GZip => Command::new("zcat"), + }; + let process = cmd.arg(vma_file_path).stdout(Stdio::piped()).spawn()?; + let stdout = process.stdout.expect("Failed to capture stdout"); + Box::new(BufReader::new(stdout)) + } + None => match &backup_args.vma_file_path { + Some(vma_file_path) => match File::open(vma_file_path) { + Err(why) => return Err(anyhow!("Couldn't open file: {}", why)), + Ok(file) => Box::new(BufReader::new(file)), + }, + None => Box::new(BufReader::new(stdin())), }, - None => Box::new(BufReader::new(stdin())), }; + let vma_reader = VmaReader::new(vma_file)?; - let pbs = create_pbs_backup_task(&args)?; + let pbs = create_pbs_backup_task(pbs_args, &backup_args)?; defer! { proxmox_backup_disconnect(pbs); @@ -467,10 +536,6 @@ pub fn backup_vma_to_pbs(args: BackupVmaToPbsArgs) -> Result<(), Error> { handle_pbs_error(pbs_err, "proxmox_backup_connect")?; } - println!("Connected to Proxmox Backup Server"); - - let start_transfer_time = SystemTime::now(); - upload_configs(&vma_reader, pbs)?; upload_block_devices(vma_reader, pbs)?; @@ -478,24 +543,23 @@ pub fn backup_vma_to_pbs(args: BackupVmaToPbsArgs) -> Result<(), Error> { handle_pbs_error(pbs_err, "proxmox_backup_finish")?; } - if args.notes.is_some() || args.log_file_path.is_some() { - let (client, store, request_args) = pbs_client_setup(&args)?; - - if args.log_file_path.is_some() { - upload_log(&client, &args, &store, request_args.clone())?; + if backup_args.notes.is_some() || backup_args.log_file_path.is_some() { + let (client, store, request_args) = pbs_client_setup(pbs_args, &backup_args)?; + + if let Some(log_file_path) = backup_args.log_file_path { + upload_log( + &client, + &log_file_path, + pbs_args, + &store, + request_args.clone(), + )?; } - if let Some(notes) = args.notes { - set_notes(&client, ¬es, &store, request_args)?; + if let Some(notes) = &backup_args.notes { + set_notes(&client, notes, &store, request_args)?; } } - let transfer_duration = SystemTime::now().duration_since(start_transfer_time)?; - let total_seconds = transfer_duration.as_secs(); - let minutes = total_seconds / 60; - let seconds = total_seconds % 60; - let milliseconds = transfer_duration.as_millis() % 1000; - println!("Backup finished within {minutes} minutes, {seconds} seconds and {milliseconds} ms"); - Ok(()) } -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel