From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256)
 (No client certificate requested)
 by lists.proxmox.com (Postfix) with ESMTPS id 6F7B76A2E2
 for <pbs-devel@lists.proxmox.com>; Wed, 24 Mar 2021 17:17:23 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
 by firstgate.proxmox.com (Proxmox) with ESMTP id 57E8DD070
 for <pbs-devel@lists.proxmox.com>; Wed, 24 Mar 2021 17:17:23 +0100 (CET)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
 [212.186.127.180])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits))
 (No client certificate requested)
 by firstgate.proxmox.com (Proxmox) with ESMTPS id 99352D05C
 for <pbs-devel@lists.proxmox.com>; Wed, 24 Mar 2021 17:17:21 +0100 (CET)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
 by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 5B5AB46347
 for <pbs-devel@lists.proxmox.com>; Wed, 24 Mar 2021 17:17:21 +0100 (CET)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Wed, 24 Mar 2021 17:17:19 +0100
Message-Id: <20210324161719.9344-2-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.20.1
In-Reply-To: <20210324161719.9344-1-d.csapak@proxmox.com>
References: <20210324161719.9344-1-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results:  0
 AWL 0.178 Adjusted score from AWL reputation of From: address
 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
 RCVD_IN_DNSWL_MED        -2.3 Sender listed at https://www.dnswl.org/,
 medium trust
 SPF_HELO_NONE           0.001 SPF: HELO does not publish an SPF Record
 SPF_PASS               -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [PATCH proxmox-backup v3 2/2] client/backup_writer:
 clarify backup and upload size
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
 <pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>, 
 <mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>, 
 <mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Wed, 24 Mar 2021 16:17:23 -0000

The text 'had to upload [KMG]iB' implies that this is the size we
actually had to send to the server, while in reality it is the
raw data size before compression.

Count the size of the compressed chunks and print it separately.
Split the average speed into its own line so they do not get too long.

Rename 'uploaded' into 'size_dirty' and 'vsize_h' into 'size'

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
changes in v3:
* split from single patch

 src/client/backup_writer.rs | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/client/backup_writer.rs b/src/client/backup_writer.rs
index 9b1cdd67..1e54d39d 100644
--- a/src/client/backup_writer.rs
+++ b/src/client/backup_writer.rs
@@ -1,6 +1,6 @@
 use std::collections::HashSet;
 use std::os::unix::fs::OpenOptionsExt;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 
 use anyhow::{bail, format_err, Error};
@@ -52,6 +52,7 @@ struct UploadStats {
     chunk_reused: usize,
     size: usize,
     size_reused: usize,
+    size_compressed: usize,
     duration: std::time::Duration,
     csum: [u8; 32],
 }
@@ -327,8 +328,8 @@ impl BackupWriter {
         )
         .await?;
 
-        let uploaded = upload_stats.size - upload_stats.size_reused;
-        let vsize_h: HumanByte = upload_stats.size.into();
+        let size_dirty = upload_stats.size - upload_stats.size_reused;
+        let size: HumanByte = upload_stats.size.into();
         let archive = if self.verbose {
             archive_name.to_string()
         } else {
@@ -336,18 +337,20 @@ impl BackupWriter {
         };
         if archive_name != CATALOG_NAME {
             let speed: HumanByte =
-                ((uploaded * 1_000_000) / (upload_stats.duration.as_micros() as usize)).into();
-            let uploaded: HumanByte = uploaded.into();
+                ((size_dirty * 1_000_000) / (upload_stats.duration.as_micros() as usize)).into();
+            let size_dirty: HumanByte = size_dirty.into();
+            let size_compressed: HumanByte = upload_stats.size_compressed.into();
             println!(
-                "{}: had to upload {} of {} in {:.2}s, average speed {}/s).",
+                "{}: had to backup {} of {} (compressed {}) in {:.2}s",
                 archive,
-                uploaded,
-                vsize_h,
-                upload_stats.duration.as_secs_f64(),
-                speed
+                size_dirty,
+                size,
+                size_compressed,
+                upload_stats.duration.as_secs_f64()
             );
+            println!("{}: average backup speed: {}/s", archive, speed);
         } else {
-            println!("Uploaded backup catalog ({})", vsize_h);
+            println!("Uploaded backup catalog ({})", size);
         }
 
         if upload_stats.size_reused > 0 && upload_stats.size > 1024 * 1024 {
@@ -633,6 +636,8 @@ impl BackupWriter {
 
         let stream_len = Arc::new(AtomicUsize::new(0));
         let stream_len2 = stream_len.clone();
+        let compressed_stream_len = Arc::new(AtomicU64::new(0));
+        let compressed_stream_len2 = compressed_stream_len.clone();
         let reused_len = Arc::new(AtomicUsize::new(0));
         let reused_len2 = reused_len.clone();
 
@@ -680,8 +685,10 @@ impl BackupWriter {
                     reused_len.fetch_add(chunk_len, Ordering::SeqCst);
                     future::ok(MergedChunkInfo::Known(vec![(offset, *digest)]))
                 } else {
+                    let compressed_stream_len2 = compressed_stream_len.clone();
                     known_chunks.insert(*digest);
                     future::ready(chunk_builder.build().map(move |(chunk, digest)| {
+                        compressed_stream_len2.fetch_add(chunk.raw_size(), Ordering::SeqCst);
                         MergedChunkInfo::New(ChunkInfo {
                             chunk,
                             digest,
@@ -754,6 +761,7 @@ impl BackupWriter {
                 let chunk_reused = known_chunk_count2.load(Ordering::SeqCst);
                 let size = stream_len2.load(Ordering::SeqCst);
                 let size_reused = reused_len2.load(Ordering::SeqCst);
+                let size_compressed = compressed_stream_len2.load(Ordering::SeqCst) as usize;
 
                 let mut guard = index_csum_2.lock().unwrap();
                 let csum = guard.take().unwrap().finish();
@@ -763,6 +771,7 @@ impl BackupWriter {
                     chunk_reused,
                     size,
                     size_reused,
+                    size_compressed,
                     duration,
                     csum,
                 })
-- 
2.20.1