From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256)
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 8735B7724C
for <pbs-devel@lists.proxmox.com>; Tue, 20 Jul 2021 11:56:40 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 76C542E092
for <pbs-devel@lists.proxmox.com>; Tue, 20 Jul 2021 11:56:40 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[94.136.29.106])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256)
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id D36322E088
for <pbs-devel@lists.proxmox.com>; Tue, 20 Jul 2021 11:56:39 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id A16FE41B3F
for <pbs-devel@lists.proxmox.com>; Tue, 20 Jul 2021 11:56:39 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Tue, 20 Jul 2021 11:56:37 +0200
Message-Id: <20210720095638.2645259-1-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.30.2
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.555 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [RFC PATCH proxmox-backup 1/2] tape: media_catalog:
commit as much as possible
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Tue, 20 Jul 2021 09:56:40 -0000
record when we finish an archive in the catalogs pending data,
and commit data until there. This way we can commit the catalog
even if we do not finish a chunk archive.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
sending as RFC, since i am not completely sure if i thought this
through completely, but in my tests it worked..
src/tape/media_catalog.rs | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs
index 65b52a42..8dacf986 100644
--- a/src/tape/media_catalog.rs
+++ b/src/tape/media_catalog.rs
@@ -66,6 +66,7 @@ pub struct MediaCatalog {
content: HashMap<String, DatastoreContent>,
+ finished_archive_len: usize,
pending: Vec<u8>,
}
@@ -223,6 +224,7 @@ impl MediaCatalog {
current_archive: None,
last_entry: None,
content: HashMap::new(),
+ finished_archive_len: 0,
pending: Vec::new(),
};
@@ -231,6 +233,7 @@ impl MediaCatalog {
if !found_magic_number {
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
}
+ me.finished_archive_len = me.pending.len();
if write {
me.file = Some(file);
@@ -300,12 +303,15 @@ impl MediaCatalog {
current_archive: None,
last_entry: None,
content: HashMap::new(),
+ finished_archive_len: 0,
pending: Vec::new(),
};
me.log_to_stdout = log_to_stdout;
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
+ me.finished_archive_len = me.pending.len();
+
me.register_label(&media_id.label.uuid, 0, 0)?;
@@ -366,31 +372,34 @@ impl MediaCatalog {
///
/// Fixme: this should be atomic ...
pub fn commit(&mut self) -> Result<(), Error> {
-
- if self.pending.is_empty() {
+ if self.finished_archive_len == 0 {
return Ok(());
}
match self.file {
Some(ref mut file) => {
- file.write_all(&self.pending)?;
+ file.write_all(&self.pending[0..self.finished_archive_len])?;
file.flush()?;
file.sync_data()?;
}
None => bail!("media catalog not writable (opened read only)"),
}
- self.pending = Vec::new();
+ let pending = self.pending.len();
+ if self.finished_archive_len < pending {
+ self.pending
+ .copy_within(self.finished_archive_len..pending, 0);
+ }
+ self.pending
+ .truncate(self.pending.len() - self.finished_archive_len);
+ self.finished_archive_len = 0;
Ok(())
}
- /// Conditionally commit if in pending data is large (> 1Mb)
+ /// Conditionally commit if finished archives in pending data is large (> 1MiB)
pub fn commit_if_large(&mut self) -> Result<(), Error> {
- if self.current_archive.is_some() {
- bail!("can't commit catalog in the middle of an chunk archive");
- }
- if self.pending.len() > 1024*1024 {
+ if self.finished_archive_len > 1024*1024 {
self.commit()?;
}
Ok(())
@@ -498,6 +507,7 @@ impl MediaCatalog {
unsafe { self.pending.write_le_value(entry)?; }
+ self.finished_archive_len = self.pending.len();
self.last_entry = Some((uuid.clone(), file_number));
Ok(())
@@ -625,6 +635,7 @@ impl MediaCatalog {
unsafe { self.pending.write_le_value(entry)?; }
+ self.finished_archive_len = self.pending.len();
self.last_entry = Some((uuid, file_number));
}
}
@@ -688,6 +699,8 @@ impl MediaCatalog {
self.pending.push(b':');
self.pending.extend(snapshot.as_bytes());
+ self.finished_archive_len = self.pending.len();
+
let content = self.content.entry(store.to_string())
.or_insert(DatastoreContent::new());
--
2.30.2