From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id DB4B890C77 for ; Thu, 25 Jan 2024 14:27:06 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 3BD8419C99 for ; Thu, 25 Jan 2024 14:26:36 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 25 Jan 2024 14:26:31 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 56F6C492D0 for ; Thu, 25 Jan 2024 14:26:30 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 25 Jan 2024 14:25:55 +0100 Message-Id: <20240125132608.1172472-17-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240125132608.1172472-1-c.ebner@proxmox.com> References: <20240125132608.1172472-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.050 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v6 proxmox-backup 16/29] fix #3174: archiver: store ref to previous backup X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 25 Jan 2024 13:27:06 -0000 Adds the stateful information needed for accessing the previous backups information such as the index and catalog. This patch only introduces the struct and interface, by setting all occurrences to None it is not used for now. Signed-off-by: Christian Ebner --- Changes since v5: - read and decode pxar format version entry, if present pbs-client/src/pxar/create.rs | 25 ++++++++++++++++--- pbs-client/src/pxar/extract.rs | 1 + pbs-client/src/pxar/mod.rs | 2 +- pbs-client/src/pxar/tools.rs | 1 + proxmox-backup-client/src/main.rs | 1 + .../src/proxmox_restore_daemon/api.rs | 1 + pxar-bin/src/main.rs | 17 +++++++------ src/tape/file_formats/snapshot_archive.rs | 7 ++++-- 8 files changed, 41 insertions(+), 14 deletions(-) diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs index 50bba4e6..4f6ffc78 100644 --- a/pbs-client/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -24,14 +24,15 @@ use proxmox_io::vec; use proxmox_lang::c_str; use proxmox_sys::fs::{self, acl, xattr}; -use pbs_datastore::catalog::BackupCatalogWriter; +use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader}; +use pbs_datastore::dynamic_index::DynamicIndexReader; use crate::pxar::metadata::errno_is_unsupported; use crate::pxar::tools::assert_single_path_component; use crate::pxar::Flags; /// Pxar options for creating a pxar archive/stream -#[derive(Default, Clone)] +#[derive(Default)] pub struct PxarCreateOptions { /// Device/mountpoint st_dev numbers that should be included. None for no limitation. pub device_set: Option>, @@ -41,6 +42,18 @@ pub struct PxarCreateOptions { pub entries_max: usize, /// Skip lost+found directory pub skip_lost_and_found: bool, + /// Reference state for partial backups + pub previous_ref: Option, +} + +/// Contains statefull information of previous backups snapshots for partial backups +pub struct PxarPrevRef { + /// Reference index for partial backups + pub index: DynamicIndexReader, + /// Reference catalog for partial backups + pub catalog: CatalogReader, + /// Reference archive name for partial backups + pub archive_name: String, } pub fn detect_fs_type(fd: RawFd) -> Result { @@ -128,6 +141,7 @@ struct Archiver { device_set: Option>, hardlinks: HashMap, file_copy_buffer: Vec, + previous_ref: Option, } type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>; @@ -166,7 +180,11 @@ where set.insert(stat.st_dev); } - let mut encoder = Encoder::new(&mut writer, &metadata).await?; + let pxar_format_version = match options.previous_ref { + Some(_) => pxar::format::FormatVersion::Version2, + None => pxar::format::FormatVersion::default(), + }; + let mut encoder = Encoder::new(&mut writer, &metadata, pxar_format_version).await?; let mut patterns = options.patterns; @@ -192,6 +210,7 @@ where device_set, hardlinks: HashMap::new(), file_copy_buffer: vec::undefined(4 * 1024 * 1024), + previous_ref: options.previous_ref, }; archiver diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs index e7dc5b02..0ded62ac 100644 --- a/pbs-client/src/pxar/extract.rs +++ b/pbs-client/src/pxar/extract.rs @@ -269,6 +269,7 @@ where }; let extract_res = match (did_match, entry.kind()) { + (_, EntryKind::Version(_)) => Ok(()), (_, EntryKind::Directory) => { self.callback(entry.path()); diff --git a/pbs-client/src/pxar/mod.rs b/pbs-client/src/pxar/mod.rs index 14674b9b..24315f5f 100644 --- a/pbs-client/src/pxar/mod.rs +++ b/pbs-client/src/pxar/mod.rs @@ -56,7 +56,7 @@ pub(crate) mod tools; mod flags; pub use flags::Flags; -pub use create::{create_archive, PxarCreateOptions}; +pub use create::{create_archive, PxarCreateOptions, PxarPrevRef}; pub use extract::{ create_tar, create_zip, extract_archive, extract_sub_dir, extract_sub_dir_seq, ErrorHandler, OverwriteFlags, PxarExtractContext, PxarExtractOptions, diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs index 174a7351..52e025e4 100644 --- a/pbs-client/src/pxar/tools.rs +++ b/pbs-client/src/pxar/tools.rs @@ -155,6 +155,7 @@ pub fn format_multi_line_entry(entry: &Entry) -> String { let meta = entry.metadata(); let (size, link, type_name) = match entry.kind() { + EntryKind::Version(version) => (format!("{version:?}"), String::new(), "version"), EntryKind::File { size, .. } => (format!("{}", *size), String::new(), "file"), EntryKind::Appendix { total } => (format!("{total}"), String::new(), "appendix"), EntryKind::AppendixRef { diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index e5caf87d..f575ccd0 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -993,6 +993,7 @@ async fn create_backup( patterns: pattern_list.clone(), entries_max: entries_max as usize, skip_lost_and_found, + previous_ref: None, }; let upload_options = UploadOptions { diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs index fb12befa..f89b0ab4 100644 --- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs +++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs @@ -356,6 +356,7 @@ fn extract( device_set: None, patterns, skip_lost_and_found: false, + previous_ref: None, }; let pxar_writer = TokioWriter::new(writer); diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs index bc044035..9376a2c1 100644 --- a/pxar-bin/src/main.rs +++ b/pxar-bin/src/main.rs @@ -330,13 +330,6 @@ async fn create_archive( Some(HashSet::new()) }; - let options = pbs_client::pxar::PxarCreateOptions { - entries_max: entries_max as usize, - device_set, - patterns, - skip_lost_and_found: false, - }; - let source = PathBuf::from(source); let dir = nix::dir::Dir::open( @@ -349,7 +342,7 @@ async fn create_archive( .create_new(true) .write(true) .mode(0o640) - .open(archive)?; + .open(archive.clone())?; let writer = std::io::BufWriter::with_capacity(1024 * 1024, file); let mut feature_flags = Flags::DEFAULT; @@ -372,6 +365,14 @@ async fn create_archive( feature_flags.remove(Flags::WITH_SOCKETS); } + let options = pbs_client::pxar::PxarCreateOptions { + entries_max: entries_max as usize, + device_set, + patterns, + skip_lost_and_found: false, + previous_ref: None, + }; + let writer = pxar::encoder::sync::StandardWriter::new(writer); pbs_client::pxar::create_archive( dir, diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs index 252384b5..abeefbd3 100644 --- a/src/tape/file_formats/snapshot_archive.rs +++ b/src/tape/file_formats/snapshot_archive.rs @@ -58,8 +58,11 @@ pub fn tape_write_snapshot_archive<'a>( )); } - let mut encoder = - pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata)?; + let mut encoder = pxar::encoder::sync::Encoder::new( + PxarTapeWriter::new(writer), + &root_metadata, + pxar::format::FormatVersion::default(), + )?; for filename in file_list.iter() { let mut file = snapshot_reader.open_file(filename).map_err(|err| { -- 2.39.2