From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 786077782B for ; Wed, 21 Jul 2021 13:18:41 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6BC07139DC for ; Wed, 21 Jul 2021 13:18:41 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 6DD39139BB for ; Wed, 21 Jul 2021 13:18:40 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 35D3D41A6D for ; Wed, 21 Jul 2021 13:18:34 +0200 (CEST) Date: Wed, 21 Jul 2021 13:18:31 +0200 (CEST) From: Dietmar Maurer To: Proxmox Backup Server development discussion , Dominik Csapak Message-ID: <482430901.322.1626866312005@webmail.proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit X-Priority: 3 Importance: Normal X-Mailer: Open-Xchange Mailer v7.10.5-Rev16 X-Originating-Client: open-xchange-appsuite X-SPAM-LEVEL: Spam detection results: 0 AWL 0.846 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox.com] Subject: Re: [pbs-devel] [RFC PATCH proxmox-backup 1/2] tape: media_catalog: commit as much as possible X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 21 Jul 2021 11:18:41 -0000 this looks clumsy. Instead, can we make catalog.start_chunk_archive and catalog.end_chunk_archive() private, and add a new method to register whole archives (catalog.register_chunk_archive()) That way all pending changes are valid and can be commited? > On 07/20/2021 11:56 AM Dominik Csapak wrote: > > > record when we finish an archive in the catalogs pending data, > and commit data until there. This way we can commit the catalog > even if we do not finish a chunk archive. > > Signed-off-by: Dominik Csapak > --- > sending as RFC, since i am not completely sure if i thought this > through completely, but in my tests it worked.. > > src/tape/media_catalog.rs | 31 ++++++++++++++++++++++--------- > 1 file changed, 22 insertions(+), 9 deletions(-) > > diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs > index 65b52a42..8dacf986 100644 > --- a/src/tape/media_catalog.rs > +++ b/src/tape/media_catalog.rs > @@ -66,6 +66,7 @@ pub struct MediaCatalog { > > content: HashMap, > > + finished_archive_len: usize, > pending: Vec, > } > > @@ -223,6 +224,7 @@ impl MediaCatalog { > current_archive: None, > last_entry: None, > content: HashMap::new(), > + finished_archive_len: 0, > pending: Vec::new(), > }; > > @@ -231,6 +233,7 @@ impl MediaCatalog { > if !found_magic_number { > me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1); > } > + me.finished_archive_len = me.pending.len(); > > if write { > me.file = Some(file); > @@ -300,12 +303,15 @@ impl MediaCatalog { > current_archive: None, > last_entry: None, > content: HashMap::new(), > + finished_archive_len: 0, > pending: Vec::new(), > }; > > me.log_to_stdout = log_to_stdout; > > me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1); > + me.finished_archive_len = me.pending.len(); > + > > me.register_label(&media_id.label.uuid, 0, 0)?; > > @@ -366,31 +372,34 @@ impl MediaCatalog { > /// > /// Fixme: this should be atomic ... > pub fn commit(&mut self) -> Result<(), Error> { > - > - if self.pending.is_empty() { > + if self.finished_archive_len == 0 { > return Ok(()); > } > > match self.file { > Some(ref mut file) => { > - file.write_all(&self.pending)?; > + file.write_all(&self.pending[0..self.finished_archive_len])?; > file.flush()?; > file.sync_data()?; > } > None => bail!("media catalog not writable (opened read only)"), > } > > - self.pending = Vec::new(); > + let pending = self.pending.len(); > + if self.finished_archive_len < pending { > + self.pending > + .copy_within(self.finished_archive_len..pending, 0); > + } > + self.pending > + .truncate(self.pending.len() - self.finished_archive_len); > + self.finished_archive_len = 0; > > Ok(()) > } > > - /// Conditionally commit if in pending data is large (> 1Mb) > + /// Conditionally commit if finished archives in pending data is large (> 1MiB) > pub fn commit_if_large(&mut self) -> Result<(), Error> { > - if self.current_archive.is_some() { > - bail!("can't commit catalog in the middle of an chunk archive"); > - } > - if self.pending.len() > 1024*1024 { > + if self.finished_archive_len > 1024*1024 { > self.commit()?; > } > Ok(()) > @@ -498,6 +507,7 @@ impl MediaCatalog { > > unsafe { self.pending.write_le_value(entry)?; } > > + self.finished_archive_len = self.pending.len(); > self.last_entry = Some((uuid.clone(), file_number)); > > Ok(()) > @@ -625,6 +635,7 @@ impl MediaCatalog { > > unsafe { self.pending.write_le_value(entry)?; } > > + self.finished_archive_len = self.pending.len(); > self.last_entry = Some((uuid, file_number)); > } > } > @@ -688,6 +699,8 @@ impl MediaCatalog { > self.pending.push(b':'); > self.pending.extend(snapshot.as_bytes()); > > + self.finished_archive_len = self.pending.len(); > + > let content = self.content.entry(store.to_string()) > .or_insert(DatastoreContent::new()); > > -- > 2.30.2 > > > > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel