From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 7398591E98 for ; Thu, 4 Apr 2024 17:05:09 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 542A3569A for ; Thu, 4 Apr 2024 17:04:39 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 4 Apr 2024 17:04:37 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8CA7C450E8 for ; Thu, 4 Apr 2024 17:04:37 +0200 (CEST) Date: Thu, 04 Apr 2024 17:04:30 +0200 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion References: <20240328123707.336951-1-c.ebner@proxmox.com> <20240328123707.336951-45-c.ebner@proxmox.com> In-Reply-To: <20240328123707.336951-45-c.ebner@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.16.0 (https://github.com/astroidmail/astroid) Message-Id: <1712242945.sq50yt5i6b.astroid@yuna.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.058 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH v3 proxmox-backup 44/58] client: pxar: add previous reference to archiver X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 04 Apr 2024 15:05:09 -0000 On March 28, 2024 1:36 pm, Christian Ebner wrote: > Read the previous snaphosts manifest and check if a split archive > with the same name is given. If so, create the accessor instance to > read the previous archive entries to be able to lookup and compare > the metata for the entries, allowing to make a decision if the > entry is reusable or not. >=20 > Signed-off-by: Christian Ebner > --- > changes since version 2: > - renamed accessor to previous metadata_accessor > - get backup reader for previous snapshot after creating the writer > instance for the new snapshot > - adapted to only use metadata mode for all or non of the given archives >=20 > pbs-client/src/pxar/create.rs | 55 ++++++++++++++++--- > proxmox-backup-client/src/main.rs | 51 ++++++++++++++++- > .../src/proxmox_restore_daemon/api.rs | 1 + > pxar-bin/src/main.rs | 1 + > 4 files changed, 97 insertions(+), 11 deletions(-) >=20 > diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.r= s > index 95a91a59b..79925bba2 100644 > --- a/pbs-client/src/pxar/create.rs > +++ b/pbs-client/src/pxar/create.rs > @@ -19,7 +19,7 @@ use nix::sys::stat::{FileStat, Mode}; > use pathpatterns::{MatchEntry, MatchFlag, MatchList, MatchType, PatternF= lag}; > use pbs_datastore::index::IndexFile; > use proxmox_sys::error::SysError; > -use pxar::accessor::aio::Accessor; > +use pxar::accessor::aio::{Accessor, Directory}; > use pxar::encoder::{LinkOffset, PayloadOffset, SeqWrite}; > use pxar::Metadata; > =20 > @@ -159,7 +159,7 @@ impl ReusedChunks { > } > =20 > /// Pxar options for creating a pxar archive/stream > -#[derive(Default, Clone)] > +#[derive(Default)] > pub struct PxarCreateOptions { > /// Device/mountpoint st_dev numbers that should be included. None f= or no limitation. > pub device_set: Option>, > @@ -171,6 +171,8 @@ pub struct PxarCreateOptions { > pub skip_lost_and_found: bool, > /// Skip xattrs of files that return E2BIG error > pub skip_e2big_xattr: bool, > + /// Reference state for partial backups > + pub previous_ref: Option, > } > =20 > /// Statefull information of previous backups snapshots for partial back= ups > @@ -270,6 +272,7 @@ struct Archiver { > file_copy_buffer: Vec, > skip_e2big_xattr: bool, > reused_chunks: ReusedChunks, > + previous_payload_index: Option, > forced_boundaries: Option>>>, > } > =20 > @@ -346,6 +349,15 @@ where > MatchType::Exclude, > )?); > } > + let (previous_payload_index, previous_metadata_accessor) =3D > + if let Some(refs) =3D options.previous_ref { > + ( > + Some(refs.payload_index), > + refs.accessor.open_root().await.ok(), > + ) > + } else { > + (None, None) > + }; > =20 > let mut archiver =3D Archiver { > feature_flags, > @@ -363,11 +375,12 @@ where > file_copy_buffer: vec::undefined(4 * 1024 * 1024), > skip_e2big_xattr: options.skip_e2big_xattr, > reused_chunks: ReusedChunks::new(), > + previous_payload_index, > forced_boundaries, > }; > =20 > archiver > - .archive_dir_contents(&mut encoder, source_dir, true) > + .archive_dir_contents(&mut encoder, previous_metadata_accessor, = source_dir, true) > .await?; > encoder.finish().await?; > encoder.close().await?; > @@ -399,6 +412,7 @@ impl Archiver { > fn archive_dir_contents<'a, T: SeqWrite + Send>( > &'a mut self, > encoder: &'a mut Encoder<'_, T>, > + mut previous_metadata_accessor: Option>>, > mut dir: Dir, > is_root: bool, > ) -> BoxFuture<'a, Result<(), Error>> { > @@ -433,9 +447,15 @@ impl Archiver { > =20 > (self.callback)(&file_entry.path)?; > self.path =3D file_entry.path; > - self.add_entry(encoder, dir_fd, &file_entry.name, &file_= entry.stat) > - .await > - .map_err(|err| self.wrap_err(err))?; > + self.add_entry( > + encoder, > + &mut previous_metadata_accessor, > + dir_fd, > + &file_entry.name, > + &file_entry.stat, > + ) > + .await > + .map_err(|err| self.wrap_err(err))?; > } > self.path =3D old_path; > self.entry_counter =3D entry_counter; > @@ -683,6 +703,7 @@ impl Archiver { > async fn add_entry( > &mut self, > encoder: &mut Encoder<'_, T>, > + previous_metadata: &mut Option>>, > parent: RawFd, > c_file_name: &CStr, > stat: &FileStat, > @@ -772,7 +793,14 @@ impl Archiver { > catalog.lock().unwrap().start_directory(c_file_name)= ?; > } > let result =3D self > - .add_directory(encoder, dir, c_file_name, &metadata,= stat) > + .add_directory( > + encoder, > + previous_metadata, > + dir, > + c_file_name, > + &metadata, > + stat, > + ) > .await; > if let Some(ref catalog) =3D self.catalog { > catalog.lock().unwrap().end_directory()?; > @@ -825,6 +853,7 @@ impl Archiver { > async fn add_directory( > &mut self, > encoder: &mut Encoder<'_, T>, > + previous_metadata_accessor: &mut Option>>, > dir: Dir, > dir_name: &CStr, > metadata: &Metadata, > @@ -855,7 +884,17 @@ impl Archiver { > log::info!("skipping mount point: {:?}", self.path); > Ok(()) > } else { > - self.archive_dir_contents(encoder, dir, false).await > + let mut dir_accessor =3D None; > + if let Some(accessor) =3D previous_metadata_accessor.as_mut(= ) { > + if let Some(file_entry) =3D accessor.lookup(dir_name).aw= ait? { > + if file_entry.entry().is_dir() { > + let dir =3D file_entry.enter_directory().await?; > + dir_accessor =3D Some(dir); > + } > + } > + } > + self.archive_dir_contents(encoder, dir_accessor, dir, false) > + .await > }; > =20 > self.fs_magic =3D old_fs_magic; > diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/sr= c/main.rs > index 0b747453c..66dcaa63e 100644 > --- a/proxmox-backup-client/src/main.rs > +++ b/proxmox-backup-client/src/main.rs > @@ -688,6 +688,10 @@ fn spawn_catalog_upload( > schema: TRAFFIC_CONTROL_BURST_SCHEMA, > optional: true, > }, > + "change-detection-mode": { > + schema: BACKUP_DETECTION_MODE_SPEC, > + optional: true, > + }, > "exclude": { > type: Array, > description: "List of paths or patterns for matching file= s to exclude.", > @@ -882,6 +886,9 @@ async fn create_backup( > =20 > let backup_time =3D backup_time_opt.unwrap_or_else(epoch_i64); > =20 > + let detection_mode =3D param["change-detection-mode"].as_str().unwra= p_or("data"); > + let detection_mode =3D parse_backup_detection_mode_specification(det= ection_mode)?; > + > let http_client =3D connect_rate_limited(&repo, rate_limit)?; > record_repository(&repo); > =20 > @@ -982,7 +989,7 @@ async fn create_backup( > None > }; > =20 > - let mut manifest =3D BackupManifest::new(snapshot); > + let mut manifest =3D BackupManifest::new(snapshot.clone()); > =20 > let mut catalog =3D None; > let mut catalog_result_rx =3D None; > @@ -1029,14 +1036,13 @@ async fn create_backup( > manifest.add_file(target, stats.size, stats.csum, crypto= .mode)?; > } > (BackupSpecificationType::PXAR, false) =3D> { > - let metadata_mode =3D false; // Until enabled via param > - > let target_base =3D if let Some(base) =3D target_base.st= rip_suffix(".pxar") { > base.to_string() > } else { > bail!("unexpected suffix in target: {target_base}"); > }; > =20 > + let metadata_mode =3D detection_mode.is_metadata(); > let (target, payload_target) =3D if metadata_mode { > ( > format!("{target_base}.mpxar.{extension}"), > @@ -1061,12 +1067,51 @@ async fn create_backup( > .unwrap() > .start_directory(std::ffi::CString::new(target.as_st= r())?.as_c_str())?; > =20 > + let mut previous_ref =3D None; > + if metadata_mode { > + if let Some(ref manifest) =3D previous_manifest { client.previous_backup_time() gives you the timestamp without the need to query all snapshots, and it's guaranteed to match the previous_manifest since both come from the same (lock-guarded) lookup when starting the writer/backup H2 session. > + let list =3D api_datastore_list_snapshots( > + &http_client, > + repo.store(), > + &backup_ns, > + Some(&snapshot.group), > + ) > + .await?; > + let mut list: Vec =3D serde_js= on::from_value(list)?; > + > + // BackupWriter::start created a new snapshot, g= et the one before > + if list.len() > 1 { > + list.sort_unstable_by(|a, b| b.backup.time.c= mp(&a.backup.time)); > + let backup_dir: BackupDir =3D > + (snapshot.group.clone(), list[1].backup.= time).into(); > + let backup_reader =3D BackupReader::start( > + &http_client, > + crypt_config.clone(), > + repo.store(), > + &backup_ns, > + &backup_dir, > + true, > + ) > + .await?; > + previous_ref =3D prepare_reference( > + &target, > + manifest.clone(), > + &client, > + backup_reader.clone(), > + crypt_config.clone(), > + ) > + .await? > + } > + } > + } > + > let pxar_options =3D pbs_client::pxar::PxarCreateOptions= { > device_set: devices.clone(), > patterns: pattern_list.clone(), > entries_max: entries_max as usize, > skip_lost_and_found, > skip_e2big_xattr, > + previous_ref, > }; > =20 > let upload_options =3D UploadOptions { > diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/p= roxmox-restore-daemon/src/proxmox_restore_daemon/api.rs > index 0883d6cda..e50cb8184 100644 > --- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs > +++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs > @@ -355,6 +355,7 @@ fn extract( > patterns, > skip_lost_and_found: false, > skip_e2big_xattr: false, > + previous_ref: None, > }; > =20 > let pxar_writer =3D TokioWriter::new(writer); > diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs > index d46c98d2b..c6d3794bb 100644 > --- a/pxar-bin/src/main.rs > +++ b/pxar-bin/src/main.rs > @@ -358,6 +358,7 @@ async fn create_archive( > patterns, > skip_lost_and_found: false, > skip_e2big_xattr: false, > + previous_ref: None, > }; > =20 > let source =3D PathBuf::from(source); > --=20 > 2.39.2 >=20 >=20 >=20 > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel >=20 >=20 >=20