From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 0D594B86DC for ; Fri, 8 Mar 2024 14:02:39 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E2E96DC68 for ; Fri, 8 Mar 2024 14:02:08 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Fri, 8 Mar 2024 14:02:05 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id EF096488CD for ; Fri, 8 Mar 2024 14:02:04 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Fri, 8 Mar 2024 14:01:49 +0100 Message-Id: <20240308130150.310352-3-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240308130150.310352-1-c.ebner@proxmox.com> References: <20240308130150.310352-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.040 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH proxmox-backup 2/3] server: sync job: include removed vanished stats X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 08 Mar 2024 13:02:39 -0000 Include statistics of vanished and therefore removed snapshots, backup groups and namespaces in the `PullStats`. In preparation for including these values in the sync jobs task log output. Signed-off-by: Christian Ebner --- src/server/pull.rs | 64 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/src/server/pull.rs b/src/server/pull.rs index d2bdd35b..14744e9c 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -65,11 +65,36 @@ pub(crate) struct LocalSource { ns: BackupNamespace, } +#[derive(Default)] +pub(crate) struct RemovedVanishedStats { + pub(crate) groups: usize, + pub(crate) snapshots: usize, + pub(crate) namespaces: usize, +} + +impl RemovedVanishedStats { + fn add(&mut self, rhs: RemovedVanishedStats) { + self.groups += rhs.groups; + self.snapshots += rhs.snapshots; + self.namespaces += rhs.namespaces; + } +} + #[derive(Default)] pub(crate) struct PullStats { pub(crate) chunk_count: usize, pub(crate) bytes: usize, pub(crate) elapsed: Duration, + pub(crate) removed: Option, +} + +impl From for PullStats { + fn from(removed: RemovedVanishedStats) -> Self { + Self { + removed: Some(removed), + ..Default::default() + } + } } impl PullStats { @@ -77,6 +102,14 @@ impl PullStats { self.chunk_count += rhs.chunk_count; self.bytes += rhs.bytes; self.elapsed += rhs.elapsed; + + if let Some(rhs_removed) = rhs.removed { + if let Some(ref mut removed) = self.removed { + removed.add(rhs_removed); + } else { + self.removed = Some(rhs_removed); + } + } } } @@ -667,6 +700,7 @@ async fn pull_index_chunks( chunk_count, bytes, elapsed, + removed: None, }) } @@ -1147,6 +1181,11 @@ async fn pull_group( .target .store .remove_backup_dir(&target_ns, snapshot.as_ref(), false)?; + pull_stats.add(PullStats::from(RemovedVanishedStats { + snapshots: 1, + groups: 0, + namespaces: 0, + })); } } @@ -1199,8 +1238,9 @@ fn check_and_remove_vanished_ns( worker: &WorkerTask, params: &PullParameters, synced_ns: HashSet, -) -> Result { +) -> Result<(bool, RemovedVanishedStats), Error> { let mut errors = false; + let mut removed_stats = RemovedVanishedStats::default(); let user_info = CachedUserInfo::new()?; // clamp like remote does so that we don't list more than we can ever have synced. @@ -1235,7 +1275,10 @@ fn check_and_remove_vanished_ns( continue; } match check_and_remove_ns(params, &local_ns) { - Ok(true) => task_log!(worker, "Removed namespace {}", local_ns), + Ok(true) => { + task_log!(worker, "Removed namespace {local_ns}"); + removed_stats.namespaces += 1; + } Ok(false) => task_log!( worker, "Did not remove namespace {} - protected snapshots remain", @@ -1248,7 +1291,7 @@ fn check_and_remove_vanished_ns( } } - Ok(errors) + Ok((errors, removed_stats)) } /// Pulls a store according to `params`. @@ -1372,7 +1415,9 @@ pub(crate) async fn pull_store( } if params.remove_vanished { - errors |= check_and_remove_vanished_ns(worker, ¶ms, synced_ns)?; + let (has_errors, stats) = check_and_remove_vanished_ns(worker, ¶ms, synced_ns)?; + errors |= has_errors; + pull_stats.add(PullStats::from(stats)); } if errors { @@ -1510,6 +1555,17 @@ pub(crate) async fn pull_ns( worker, "kept some protected snapshots of group '{local_group}'", ); + pull_stats.add(PullStats::from(RemovedVanishedStats { + snapshots: stats.removed_snapshots(), + groups: 0, + namespaces: 0, + })); + } else { + pull_stats.add(PullStats::from(RemovedVanishedStats { + snapshots: stats.removed_snapshots(), + groups: 1, + namespaces: 0, + })); } } Err(err) => { -- 2.39.2