From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 8735B7724C for ; Tue, 20 Jul 2021 11:56:40 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 76C542E092 for ; Tue, 20 Jul 2021 11:56:40 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id D36322E088 for ; Tue, 20 Jul 2021 11:56:39 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id A16FE41B3F for ; Tue, 20 Jul 2021 11:56:39 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Tue, 20 Jul 2021 11:56:37 +0200 Message-Id: <20210720095638.2645259-1-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.555 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [RFC PATCH proxmox-backup 1/2] tape: media_catalog: commit as much as possible X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 20 Jul 2021 09:56:40 -0000 record when we finish an archive in the catalogs pending data, and commit data until there. This way we can commit the catalog even if we do not finish a chunk archive. Signed-off-by: Dominik Csapak --- sending as RFC, since i am not completely sure if i thought this through completely, but in my tests it worked.. src/tape/media_catalog.rs | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs index 65b52a42..8dacf986 100644 --- a/src/tape/media_catalog.rs +++ b/src/tape/media_catalog.rs @@ -66,6 +66,7 @@ pub struct MediaCatalog { content: HashMap, + finished_archive_len: usize, pending: Vec, } @@ -223,6 +224,7 @@ impl MediaCatalog { current_archive: None, last_entry: None, content: HashMap::new(), + finished_archive_len: 0, pending: Vec::new(), }; @@ -231,6 +233,7 @@ impl MediaCatalog { if !found_magic_number { me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1); } + me.finished_archive_len = me.pending.len(); if write { me.file = Some(file); @@ -300,12 +303,15 @@ impl MediaCatalog { current_archive: None, last_entry: None, content: HashMap::new(), + finished_archive_len: 0, pending: Vec::new(), }; me.log_to_stdout = log_to_stdout; me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1); + me.finished_archive_len = me.pending.len(); + me.register_label(&media_id.label.uuid, 0, 0)?; @@ -366,31 +372,34 @@ impl MediaCatalog { /// /// Fixme: this should be atomic ... pub fn commit(&mut self) -> Result<(), Error> { - - if self.pending.is_empty() { + if self.finished_archive_len == 0 { return Ok(()); } match self.file { Some(ref mut file) => { - file.write_all(&self.pending)?; + file.write_all(&self.pending[0..self.finished_archive_len])?; file.flush()?; file.sync_data()?; } None => bail!("media catalog not writable (opened read only)"), } - self.pending = Vec::new(); + let pending = self.pending.len(); + if self.finished_archive_len < pending { + self.pending + .copy_within(self.finished_archive_len..pending, 0); + } + self.pending + .truncate(self.pending.len() - self.finished_archive_len); + self.finished_archive_len = 0; Ok(()) } - /// Conditionally commit if in pending data is large (> 1Mb) + /// Conditionally commit if finished archives in pending data is large (> 1MiB) pub fn commit_if_large(&mut self) -> Result<(), Error> { - if self.current_archive.is_some() { - bail!("can't commit catalog in the middle of an chunk archive"); - } - if self.pending.len() > 1024*1024 { + if self.finished_archive_len > 1024*1024 { self.commit()?; } Ok(()) @@ -498,6 +507,7 @@ impl MediaCatalog { unsafe { self.pending.write_le_value(entry)?; } + self.finished_archive_len = self.pending.len(); self.last_entry = Some((uuid.clone(), file_number)); Ok(()) @@ -625,6 +635,7 @@ impl MediaCatalog { unsafe { self.pending.write_le_value(entry)?; } + self.finished_archive_len = self.pending.len(); self.last_entry = Some((uuid, file_number)); } } @@ -688,6 +699,8 @@ impl MediaCatalog { self.pending.push(b':'); self.pending.extend(snapshot.as_bytes()); + self.finished_archive_len = self.pending.len(); + let content = self.content.entry(store.to_string()) .or_insert(DatastoreContent::new()); -- 2.30.2