From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id A0F31989B2 for ; Wed, 15 Nov 2023 16:48:59 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0A06B96C3 for ; Wed, 15 Nov 2023 16:48:38 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 15 Nov 2023 16:48:36 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id F0CAE432A8 for ; Wed, 15 Nov 2023 16:48:35 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Wed, 15 Nov 2023 16:48:12 +0100 Message-Id: <20231115154813.281564-28-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20231115154813.281564-1-c.ebner@proxmox.com> References: <20231115154813.281564-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.057 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v5 proxmox-backup 27/28] catalog: use format version 2 conditionally X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Nov 2023 15:48:59 -0000 Only use the catalog format version 2 when performing backups with the change detection mode set to metadata. This makes sure to remain compatible with older clients which do not support the format version 2 at least for backups created using the regular mode. Signed-off-by: Christian Ebner --- Changes since version 4: - not present in version 4 pbs-datastore/src/catalog.rs | 71 +++++++++++++++++++++++++------ proxmox-backup-client/src/main.rs | 29 ++++++++++--- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/pbs-datastore/src/catalog.rs b/pbs-datastore/src/catalog.rs index fe076a94..93457302 100644 --- a/pbs-datastore/src/catalog.rs +++ b/pbs-datastore/src/catalog.rs @@ -65,6 +65,12 @@ pub enum CatalogEntryType { Socket = b's', } +#[derive(PartialEq)] +pub enum CatalogFormatVersion { + V1, + V2, +} + impl TryFrom for CatalogEntryType { type Error = Error; @@ -556,17 +562,22 @@ pub struct CatalogWriter { writer: W, dirstack: Vec, pos: u64, + version: CatalogFormatVersion, } impl CatalogWriter { /// Create a new CatalogWriter instance - pub fn new(writer: W) -> Result { + pub fn new(writer: W, version: CatalogFormatVersion) -> Result { let mut me = Self { writer, dirstack: vec![DirInfo::new_rootdir()], pos: 0, + version, }; - me.write_all(&PROXMOX_CATALOG_FILE_MAGIC_2_0)?; + match me.version { + CatalogFormatVersion::V1 => me.write_all(&PROXMOX_CATALOG_FILE_MAGIC_1_0)?, + CatalogFormatVersion::V2 => me.write_all(&PROXMOX_CATALOG_FILE_MAGIC_2_0)?, + } Ok(me) } @@ -600,6 +611,10 @@ impl CatalogWriter { impl BackupCatalogWriter for CatalogWriter { fn start_archive(&mut self, name: &CStr) -> Result<(), Error> { + if self.version == CatalogFormatVersion::V1 { + return self.start_directory(name); + } + let new = DirInfo::new(name.to_owned()); self.dirstack.push(new); Ok(()) @@ -609,6 +624,9 @@ impl BackupCatalogWriter for CatalogWriter { &mut self, appendix: Option, ) -> Result<(), Error> { + if self.version == CatalogFormatVersion::V1 { + return self.end_directory(); + } let (start, name) = match self.dirstack.pop() { Some(dir) => { let start = self.pos; @@ -689,17 +707,32 @@ impl BackupCatalogWriter for CatalogWriter { .last_mut() .ok_or_else(|| format_err!("outside root"))?; let name = name.to_bytes().to_vec(); - let file_offset = FileOffset { - offset: file_offset.raw(), + let entry = match self.version { + CatalogFormatVersion::V1 => { + DirEntry { + name, + attr: DirEntryAttribute::File { + size, + mtime, + extension: None, + }, + } + } + CatalogFormatVersion::V2 => { + let file_offset = FileOffset { + offset: file_offset.raw(), + }; + DirEntry { + name, + attr: DirEntryAttribute::File { + size, + mtime, + extension: Some(CatalogV2Extension { ctime, file_offset }), + }, + } + } }; - dir.entries.push(DirEntry { - name, - attr: DirEntryAttribute::File { - size, - mtime, - extension: Some(CatalogV2Extension { ctime, file_offset }), - }, - }); + dir.entries.push(entry); Ok(()) } @@ -711,6 +744,9 @@ impl BackupCatalogWriter for CatalogWriter { ctime: i64, appendix_ref_offset: pxar::encoder::AppendixRefOffset, ) -> Result<(), Error> { + if self.version == CatalogFormatVersion::V1 { + bail!("unsupported by catalog format version 1"); + } let dir = self .dirstack .last_mut() @@ -857,6 +893,17 @@ impl CatalogReader { }) } + pub fn format_version(&mut self) -> Result { + self.reader.seek(SeekFrom::Start(0))?; + let mut magic = [0u8; 8]; + self.reader.read_exact(&mut magic)?; + match magic { + PROXMOX_CATALOG_FILE_MAGIC_1_0 => Ok(CatalogFormatVersion::V1), + PROXMOX_CATALOG_FILE_MAGIC_2_0 => Ok(CatalogFormatVersion::V2), + _ => bail!("got unexpected magic number for catalog"), + } + } + pub fn appendix_offset( &mut self, archive_name: &[u8], diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index 65d3308a..0a2e0c8b 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -49,7 +49,7 @@ use pbs_client::{ BackupStats, BackupWriter, ChunkStream, FixedChunkStream, HttpClient, PxarBackupStream, RemoteChunkReader, UploadOptions, BACKUP_DETECTION_MODE_SPEC, BACKUP_SOURCE_SCHEMA, }; -use pbs_datastore::catalog::{CatalogReader, CatalogWriter}; +use pbs_datastore::catalog::{CatalogFormatVersion, CatalogReader, CatalogWriter}; use pbs_datastore::chunk_store::verify_chunk_size; use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader}; use pbs_datastore::fixed_index::FixedIndexReader; @@ -536,6 +536,7 @@ struct CatalogUploadResult { fn spawn_catalog_upload( client: Arc, encrypt: bool, + catalog_format_version: CatalogFormatVersion, ) -> Result { let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes let catalog_stream = proxmox_async::blocking::StdChannelStream(catalog_rx); @@ -549,9 +550,10 @@ fn spawn_catalog_upload( injections.clone(), ); - let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new( - StdChannelWriter::new(catalog_tx), - ))?)); + let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new( + TokioWriterAdapter::new(StdChannelWriter::new(catalog_tx)), + catalog_format_version, + )?)); let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel(); @@ -1015,8 +1017,16 @@ async fn create_backup( (BackupSpecificationType::PXAR, false) => { // start catalog upload on first use if catalog.is_none() { - let catalog_upload_res = - spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?; + let catalog_format_version = if let BackupDetectionMode::Metadata(_) = detection_mode { + CatalogFormatVersion::V2 + } else { + CatalogFormatVersion::V1 + }; + let catalog_upload_res = spawn_catalog_upload( + client.clone(), + crypto.mode == CryptMode::Encrypt, + catalog_format_version, + )?; catalog = Some(catalog_upload_res.catalog_writer); catalog_result_rx = Some(catalog_upload_res.result); } @@ -1214,8 +1224,13 @@ async fn download_reference_catalog( .map_err(|err| format_err!("failed to download reference catalog - {}", err))?; catalogfile.seek(SeekFrom::Start(0))?; + let mut reader = CatalogReader::new(catalogfile); - Ok(CatalogReader::new(catalogfile)) + match reader.format_version() { + Ok(CatalogFormatVersion::V2) => Ok(reader), + Ok(CatalogFormatVersion::V1) => Err(format_err!("unsupported catalog format version")), + Err(err) => Err(err), + } } async fn dump_image( -- 2.39.2