From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9])
by lore.proxmox.com (Postfix) with ESMTPS id E95531FF389
for <inbox@lore.proxmox.com>; Tue, 7 May 2024 17:54:23 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id F3ADC12906;
Tue, 7 May 2024 17:54:16 +0200 (CEST)
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Tue, 7 May 2024 17:52:30 +0200
Message-Id: <20240507155244.793819-49-c.ebner@proxmox.com>
X-Mailer: git-send-email 2.39.2
In-Reply-To: <20240507155244.793819-1-c.ebner@proxmox.com>
References: <20240507155244.793819-1-c.ebner@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.027 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [PATCH v5 proxmox-backup 48/62] pxar: create: keep
track of reused chunks and files
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
Track and log reused or reencoded files as well as the reused chunks
and their paddings.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 4:
- no changes
pbs-client/src/pxar/create.rs | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index b2932c973..b03bd5a17 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -141,6 +141,17 @@ struct HardLinkInfo {
st_ino: u64,
}
+#[derive(Default)]
+struct ReuseStats {
+ files_reused_count: u64,
+ files_hardlink_count: u64,
+ files_reencoded_count: u64,
+ total_injected_count: u64,
+ partial_chunks_count: u64,
+ total_injected_size: u64,
+ total_reused_payload_size: u64,
+}
+
struct Archiver {
feature_flags: Flags,
fs_feature_flags: Flags,
@@ -164,6 +175,7 @@ struct Archiver {
cached_range: Range<u64>,
cached_last_chunk: Option<ReusableDynamicEntry>,
caching_enabled: bool,
+ reuse_stats: ReuseStats,
}
type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>;
@@ -273,6 +285,7 @@ where
cached_last_chunk: None,
cached_hardlinks: HashSet::new(),
caching_enabled: false,
+ reuse_stats: ReuseStats::default(),
};
archiver
@@ -802,15 +815,22 @@ impl Archiver {
}
let offset: LinkOffset = if let Some(payload_offset) = payload_offset {
+ self.reuse_stats.total_reused_payload_size +=
+ file_size + size_of::<pxar::format::Header>() as u64;
+ self.reuse_stats.files_reused_count += 1;
+
encoder
.add_payload_ref(metadata, file_name, file_size, payload_offset)
.await?
} else {
+ self.reuse_stats.files_reencoded_count += 1;
+
self.add_regular_file(encoder, fd, file_name, metadata, file_size)
.await?
};
if stat.st_nlink > 1 {
+ self.reuse_stats.files_hardlink_count += 1;
self.hardlinks
.insert(link_info, (self.path.clone(), offset));
}
@@ -1147,6 +1167,13 @@ impl Archiver {
HumanByte::from(chunk.padding),
HumanByte::from(chunk.size()),
);
+ self.reuse_stats.total_injected_size += chunk.size();
+ self.reuse_stats.total_injected_count += 1;
+
+ if chunk.padding > 0 {
+ self.reuse_stats.partial_chunks_count += 1;
+ }
+
size = size.add(chunk.size());
chunk_list.push(chunk.clone());
}
--
2.39.2
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel