From: "Fabian Grünbichler" <f.gruenbichler@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 2/4] pull: factor out interpolated progress
Date: Mon, 30 Nov 2020 16:27:19 +0100 [thread overview]
Message-ID: <20201130152721.666511-3-f.gruenbichler@proxmox.com> (raw)
In-Reply-To: <20201130152721.666511-1-f.gruenbichler@proxmox.com>
and add group/snapshot count info.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
Note: not 100% happy with struct and field naming here, very open for
better suggestions..
src/backup/datastore.rs | 47 +++++++++++++++++++++++++++++++++++++++++
src/client/pull.rs | 27 ++++++++++-------------
2 files changed, 58 insertions(+), 16 deletions(-)
diff --git a/src/backup/datastore.rs b/src/backup/datastore.rs
index 19efc23f..4f49d03f 100644
--- a/src/backup/datastore.rs
+++ b/src/backup/datastore.rs
@@ -739,3 +739,50 @@ impl DataStore {
self.verify_new
}
}
+
+#[derive(Debug, Clone, Default)]
+/// Tracker for progress of operations iterating over `Datastore` contents.
+pub struct StoreProgress {
+ /// Completed groups
+ pub done_groups: u64,
+ /// Total groups
+ pub total_groups: u64,
+ /// Completed snapshots within current group
+ pub done_snapshots: u64,
+ /// Total snapshots in current group
+ pub group_snapshots: u64,
+}
+
+impl StoreProgress {
+ pub fn new(total_groups: u64) -> Self {
+ StoreProgress {
+ total_groups,
+ .. Default::default()
+ }
+ }
+
+ /// Calculates an interpolated relative progress based on current counters.
+ pub fn percentage(&self) -> f64 {
+ let per_groups = (self.done_groups as f64) / (self.total_groups as f64);
+ if self.group_snapshots == 0 {
+ per_groups
+ } else {
+ let per_snapshots = (self.done_snapshots as f64) / (self.group_snapshots as f64);
+ per_groups + (1.0 / self.total_groups as f64) * per_snapshots
+ }
+ }
+}
+
+impl std::fmt::Display for StoreProgress {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "{:.2}% ({} of {} groups, {} of {} group snapshots)",
+ self.percentage() * 100.0,
+ self.done_groups,
+ self.total_groups,
+ self.done_snapshots,
+ self.group_snapshots,
+ )
+ }
+}
diff --git a/src/client/pull.rs b/src/client/pull.rs
index 0c9afe0a..2555a14c 100644
--- a/src/client/pull.rs
+++ b/src/client/pull.rs
@@ -395,7 +395,7 @@ pub async fn pull_group(
tgt_store: Arc<DataStore>,
group: &BackupGroup,
delete: bool,
- progress: Option<(usize, usize)>, // (groups_done, group_count)
+ progress: &mut StoreProgress,
) -> Result<(), Error> {
let path = format!("api2/json/admin/datastore/{}/snapshots", src_repo.store());
@@ -418,18 +418,10 @@ pub async fn pull_group(
let mut remote_snapshots = std::collections::HashSet::new();
- let (per_start, per_group) = if let Some((groups_done, group_count)) = progress {
- let per_start = (groups_done as f64)/(group_count as f64);
- let per_group = 1.0/(group_count as f64);
- (per_start, per_group)
- } else {
- (0.0, 1.0)
- };
-
// start with 16384 chunks (up to 65GB)
let downloaded_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*64)));
- let snapshot_count = list.len();
+ progress.group_snapshots = list.len() as u64;
for (pos, item) in list.into_iter().enumerate() {
let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
@@ -469,9 +461,8 @@ pub async fn pull_group(
let result = pull_snapshot_from(worker, reader, tgt_store.clone(), &snapshot, downloaded_chunks.clone()).await;
- let percentage = (pos as f64)/(snapshot_count as f64);
- let percentage = per_start + percentage*per_group;
- worker.log(format!("percentage done: {:.2}%", percentage*100.0));
+ progress.done_snapshots = pos as u64 + 1;
+ worker.log(format!("percentage done: {}", progress.clone()));
result?; // stop on error
}
@@ -523,9 +514,13 @@ pub async fn pull_store(
new_groups.insert(BackupGroup::new(&item.backup_type, &item.backup_id));
}
- let group_count = list.len();
+ let mut progress = StoreProgress::new(list.len() as u64);
+
+ for (done, item) in list.into_iter().enumerate() {
+ progress.done_groups = done as u64;
+ progress.done_snapshots = 0;
+ progress.group_snapshots = 0;
- for (groups_done, item) in list.into_iter().enumerate() {
let group = BackupGroup::new(&item.backup_type, &item.backup_id);
let (owner, _lock_guard) = match tgt_store.create_locked_backup_group(&group, &auth_id) {
@@ -551,7 +546,7 @@ pub async fn pull_store(
tgt_store.clone(),
&group,
delete,
- Some((groups_done, group_count)),
+ &mut progress,
).await {
worker.log(format!(
"sync group {}/{} failed - {}",
--
2.20.1
next prev parent reply other threads:[~2020-11-30 15:27 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-11-30 15:27 [pbs-devel] [PATCH proxmox-backup 0/4] pull/verify unified progress Fabian Grünbichler
2020-11-30 15:27 ` [pbs-devel] [PATCH proxmox-backup 1/4] remove BackupGroup::list_groups Fabian Grünbichler
2020-11-30 15:27 ` Fabian Grünbichler [this message]
2020-12-01 5:21 ` [pbs-devel] [PATCH proxmox-backup 2/4] pull: factor out interpolated progress Dietmar Maurer
2020-11-30 15:27 ` [pbs-devel] [PATCH proxmox-backup 3/4] progress: add format variants Fabian Grünbichler
2020-11-30 15:27 ` [pbs-devel] [PATCH proxmox-backup 4/4] verify: use same progress as pull Fabian Grünbichler
2020-12-01 5:40 ` [pbs-devel] applied: [PATCH proxmox-backup 0/4] pull/verify unified progress Dietmar Maurer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201130152721.666511-3-f.gruenbichler@proxmox.com \
--to=f.gruenbichler@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox