From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 345736197E for ; Mon, 30 Nov 2020 16:27:44 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 2C30918020 for ; Mon, 30 Nov 2020 16:27:44 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 8A85D18016 for ; Mon, 30 Nov 2020 16:27:43 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 58014446BC for ; Mon, 30 Nov 2020 16:27:43 +0100 (CET) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Mon, 30 Nov 2020 16:27:19 +0100 Message-Id: <20201130152721.666511-3-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201130152721.666511-1-f.gruenbichler@proxmox.com> References: <20201130152721.666511-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.024 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 2/4] pull: factor out interpolated progress X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 30 Nov 2020 15:27:44 -0000 and add group/snapshot count info. Signed-off-by: Fabian Grünbichler --- Note: not 100% happy with struct and field naming here, very open for better suggestions.. src/backup/datastore.rs | 47 +++++++++++++++++++++++++++++++++++++++++ src/client/pull.rs | 27 ++++++++++------------- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/backup/datastore.rs b/src/backup/datastore.rs index 19efc23f..4f49d03f 100644 --- a/src/backup/datastore.rs +++ b/src/backup/datastore.rs @@ -739,3 +739,50 @@ impl DataStore { self.verify_new } } + +#[derive(Debug, Clone, Default)] +/// Tracker for progress of operations iterating over `Datastore` contents. +pub struct StoreProgress { + /// Completed groups + pub done_groups: u64, + /// Total groups + pub total_groups: u64, + /// Completed snapshots within current group + pub done_snapshots: u64, + /// Total snapshots in current group + pub group_snapshots: u64, +} + +impl StoreProgress { + pub fn new(total_groups: u64) -> Self { + StoreProgress { + total_groups, + .. Default::default() + } + } + + /// Calculates an interpolated relative progress based on current counters. + pub fn percentage(&self) -> f64 { + let per_groups = (self.done_groups as f64) / (self.total_groups as f64); + if self.group_snapshots == 0 { + per_groups + } else { + let per_snapshots = (self.done_snapshots as f64) / (self.group_snapshots as f64); + per_groups + (1.0 / self.total_groups as f64) * per_snapshots + } + } +} + +impl std::fmt::Display for StoreProgress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:.2}% ({} of {} groups, {} of {} group snapshots)", + self.percentage() * 100.0, + self.done_groups, + self.total_groups, + self.done_snapshots, + self.group_snapshots, + ) + } +} diff --git a/src/client/pull.rs b/src/client/pull.rs index 0c9afe0a..2555a14c 100644 --- a/src/client/pull.rs +++ b/src/client/pull.rs @@ -395,7 +395,7 @@ pub async fn pull_group( tgt_store: Arc, group: &BackupGroup, delete: bool, - progress: Option<(usize, usize)>, // (groups_done, group_count) + progress: &mut StoreProgress, ) -> Result<(), Error> { let path = format!("api2/json/admin/datastore/{}/snapshots", src_repo.store()); @@ -418,18 +418,10 @@ pub async fn pull_group( let mut remote_snapshots = std::collections::HashSet::new(); - let (per_start, per_group) = if let Some((groups_done, group_count)) = progress { - let per_start = (groups_done as f64)/(group_count as f64); - let per_group = 1.0/(group_count as f64); - (per_start, per_group) - } else { - (0.0, 1.0) - }; - // start with 16384 chunks (up to 65GB) let downloaded_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*64))); - let snapshot_count = list.len(); + progress.group_snapshots = list.len() as u64; for (pos, item) in list.into_iter().enumerate() { let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?; @@ -469,9 +461,8 @@ pub async fn pull_group( let result = pull_snapshot_from(worker, reader, tgt_store.clone(), &snapshot, downloaded_chunks.clone()).await; - let percentage = (pos as f64)/(snapshot_count as f64); - let percentage = per_start + percentage*per_group; - worker.log(format!("percentage done: {:.2}%", percentage*100.0)); + progress.done_snapshots = pos as u64 + 1; + worker.log(format!("percentage done: {}", progress.clone())); result?; // stop on error } @@ -523,9 +514,13 @@ pub async fn pull_store( new_groups.insert(BackupGroup::new(&item.backup_type, &item.backup_id)); } - let group_count = list.len(); + let mut progress = StoreProgress::new(list.len() as u64); + + for (done, item) in list.into_iter().enumerate() { + progress.done_groups = done as u64; + progress.done_snapshots = 0; + progress.group_snapshots = 0; - for (groups_done, item) in list.into_iter().enumerate() { let group = BackupGroup::new(&item.backup_type, &item.backup_id); let (owner, _lock_guard) = match tgt_store.create_locked_backup_group(&group, &auth_id) { @@ -551,7 +546,7 @@ pub async fn pull_store( tgt_store.clone(), &group, delete, - Some((groups_done, group_count)), + &mut progress, ).await { worker.log(format!( "sync group {}/{} failed - {}", -- 2.20.1