Re: [pbs-devel] [RFC PATCH proxmox-backup 1/2] tape: media_catalog: commit as much as possible

* Re: [pbs-devel] [RFC PATCH proxmox-backup 1/2] tape: media_catalog: commit as much as possible
@ 2021-07-21 11:18 Dietmar Maurer
  0 siblings, 0 replies; 2+ messages in thread
From: Dietmar Maurer @ 2021-07-21 11:18 UTC (permalink / raw)
  To: Proxmox Backup Server development discussion, Dominik Csapak

this looks clumsy.

Instead, can we make catalog.start_chunk_archive and catalog.end_chunk_archive() private,
and add a new method to register whole archives (catalog.register_chunk_archive())

That way all pending changes are valid and can be commited?

> On 07/20/2021 11:56 AM Dominik Csapak <d.csapak@proxmox.com> wrote:
> 
>  
> record when we finish an archive in the catalogs pending data,
> and commit data until there. This way we can commit the catalog
> even if we do not finish a chunk archive.
> 
> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
> ---
> sending as RFC, since i am not completely sure if i thought this
> through completely, but in my tests it worked..
> 
>  src/tape/media_catalog.rs | 31 ++++++++++++++++++++++---------
>  1 file changed, 22 insertions(+), 9 deletions(-)
> 
> diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs
> index 65b52a42..8dacf986 100644
> --- a/src/tape/media_catalog.rs
> +++ b/src/tape/media_catalog.rs
> @@ -66,6 +66,7 @@ pub struct MediaCatalog  {
>  
>      content: HashMap<String, DatastoreContent>,
>  
> +    finished_archive_len: usize,
>      pending: Vec<u8>,
>  }
>  
> @@ -223,6 +224,7 @@ impl MediaCatalog {
>                  current_archive: None,
>                  last_entry: None,
>                  content: HashMap::new(),
> +                finished_archive_len: 0,
>                  pending: Vec::new(),
>              };
>  
> @@ -231,6 +233,7 @@ impl MediaCatalog {
>              if !found_magic_number {
>                  me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
>              }
> +            me.finished_archive_len = me.pending.len();
>  
>              if write {
>                  me.file = Some(file);
> @@ -300,12 +303,15 @@ impl MediaCatalog {
>                  current_archive: None,
>                  last_entry: None,
>                  content: HashMap::new(),
> +                finished_archive_len: 0,
>                  pending: Vec::new(),
>              };
>  
>              me.log_to_stdout = log_to_stdout;
>  
>              me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
> +            me.finished_archive_len = me.pending.len();
> +
>  
>              me.register_label(&media_id.label.uuid, 0, 0)?;
>  
> @@ -366,31 +372,34 @@ impl MediaCatalog {
>      ///
>      /// Fixme: this should be atomic ...
>      pub fn commit(&mut self) -> Result<(), Error> {
> -
> -        if self.pending.is_empty() {
> +        if self.finished_archive_len == 0 {
>              return Ok(());
>          }
>  
>          match self.file {
>              Some(ref mut file) => {
> -                file.write_all(&self.pending)?;
> +                file.write_all(&self.pending[0..self.finished_archive_len])?;
>                  file.flush()?;
>                  file.sync_data()?;
>              }
>              None => bail!("media catalog not writable (opened read only)"),
>          }
>  
> -        self.pending = Vec::new();
> +        let pending = self.pending.len();
> +        if self.finished_archive_len < pending {
> +            self.pending
> +                .copy_within(self.finished_archive_len..pending, 0);
> +        }
> +        self.pending
> +            .truncate(self.pending.len() - self.finished_archive_len);
> +        self.finished_archive_len = 0;
>  
>          Ok(())
>      }
>  
> -    /// Conditionally commit if in pending data is large (> 1Mb)
> +    /// Conditionally commit if finished archives in pending data is large (> 1MiB)
>      pub fn commit_if_large(&mut self) -> Result<(), Error> {
> -        if self.current_archive.is_some() {
> -            bail!("can't commit catalog in the middle of an chunk archive");
> -        }
> -        if self.pending.len() > 1024*1024 {
> +        if self.finished_archive_len > 1024*1024 {
>              self.commit()?;
>          }
>          Ok(())
> @@ -498,6 +507,7 @@ impl MediaCatalog {
>  
>          unsafe { self.pending.write_le_value(entry)?; }
>  
> +        self.finished_archive_len = self.pending.len();
>          self.last_entry = Some((uuid.clone(), file_number));
>  
>          Ok(())
> @@ -625,6 +635,7 @@ impl MediaCatalog {
>  
>                  unsafe { self.pending.write_le_value(entry)?; }
>  
> +                self.finished_archive_len = self.pending.len();
>                  self.last_entry = Some((uuid, file_number));
>              }
>          }
> @@ -688,6 +699,8 @@ impl MediaCatalog {
>          self.pending.push(b':');
>          self.pending.extend(snapshot.as_bytes());
>  
> +        self.finished_archive_len = self.pending.len();
> +
>          let content = self.content.entry(store.to_string())
>              .or_insert(DatastoreContent::new());
>  
> -- 
> 2.30.2
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

^ permalink raw reply	[flat|nested] 2+ messages in thread