all lists on lists.proxmox.com
 help / color / mirror / Atom feed
* [pbs-devel] [RFC PATCH proxmox-backup 1/2] tape: media_catalog: commit as much as possible
@ 2021-07-20  9:56 Dominik Csapak
  2021-07-20  9:56 ` [pbs-devel] [RFC PATCH promxox-backup 2/2] api2: tape/backup: commit pool_writer even on error Dominik Csapak
  0 siblings, 1 reply; 2+ messages in thread
From: Dominik Csapak @ 2021-07-20  9:56 UTC (permalink / raw)
  To: pbs-devel

record when we finish an archive in the catalogs pending data,
and commit data until there. This way we can commit the catalog
even if we do not finish a chunk archive.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
sending as RFC, since i am not completely sure if i thought this
through completely, but in my tests it worked..

 src/tape/media_catalog.rs | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs
index 65b52a42..8dacf986 100644
--- a/src/tape/media_catalog.rs
+++ b/src/tape/media_catalog.rs
@@ -66,6 +66,7 @@ pub struct MediaCatalog  {
 
     content: HashMap<String, DatastoreContent>,
 
+    finished_archive_len: usize,
     pending: Vec<u8>,
 }
 
@@ -223,6 +224,7 @@ impl MediaCatalog {
                 current_archive: None,
                 last_entry: None,
                 content: HashMap::new(),
+                finished_archive_len: 0,
                 pending: Vec::new(),
             };
 
@@ -231,6 +233,7 @@ impl MediaCatalog {
             if !found_magic_number {
                 me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
             }
+            me.finished_archive_len = me.pending.len();
 
             if write {
                 me.file = Some(file);
@@ -300,12 +303,15 @@ impl MediaCatalog {
                 current_archive: None,
                 last_entry: None,
                 content: HashMap::new(),
+                finished_archive_len: 0,
                 pending: Vec::new(),
             };
 
             me.log_to_stdout = log_to_stdout;
 
             me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
+            me.finished_archive_len = me.pending.len();
+
 
             me.register_label(&media_id.label.uuid, 0, 0)?;
 
@@ -366,31 +372,34 @@ impl MediaCatalog {
     ///
     /// Fixme: this should be atomic ...
     pub fn commit(&mut self) -> Result<(), Error> {
-
-        if self.pending.is_empty() {
+        if self.finished_archive_len == 0 {
             return Ok(());
         }
 
         match self.file {
             Some(ref mut file) => {
-                file.write_all(&self.pending)?;
+                file.write_all(&self.pending[0..self.finished_archive_len])?;
                 file.flush()?;
                 file.sync_data()?;
             }
             None => bail!("media catalog not writable (opened read only)"),
         }
 
-        self.pending = Vec::new();
+        let pending = self.pending.len();
+        if self.finished_archive_len < pending {
+            self.pending
+                .copy_within(self.finished_archive_len..pending, 0);
+        }
+        self.pending
+            .truncate(self.pending.len() - self.finished_archive_len);
+        self.finished_archive_len = 0;
 
         Ok(())
     }
 
-    /// Conditionally commit if in pending data is large (> 1Mb)
+    /// Conditionally commit if finished archives in pending data is large (> 1MiB)
     pub fn commit_if_large(&mut self) -> Result<(), Error> {
-        if self.current_archive.is_some() {
-            bail!("can't commit catalog in the middle of an chunk archive");
-        }
-        if self.pending.len() > 1024*1024 {
+        if self.finished_archive_len > 1024*1024 {
             self.commit()?;
         }
         Ok(())
@@ -498,6 +507,7 @@ impl MediaCatalog {
 
         unsafe { self.pending.write_le_value(entry)?; }
 
+        self.finished_archive_len = self.pending.len();
         self.last_entry = Some((uuid.clone(), file_number));
 
         Ok(())
@@ -625,6 +635,7 @@ impl MediaCatalog {
 
                 unsafe { self.pending.write_le_value(entry)?; }
 
+                self.finished_archive_len = self.pending.len();
                 self.last_entry = Some((uuid, file_number));
             }
         }
@@ -688,6 +699,8 @@ impl MediaCatalog {
         self.pending.push(b':');
         self.pending.extend(snapshot.as_bytes());
 
+        self.finished_archive_len = self.pending.len();
+
         let content = self.content.entry(store.to_string())
             .or_insert(DatastoreContent::new());
 
-- 
2.30.2





^ permalink raw reply	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2021-07-20  9:57 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-07-20  9:56 [pbs-devel] [RFC PATCH proxmox-backup 1/2] tape: media_catalog: commit as much as possible Dominik Csapak
2021-07-20  9:56 ` [pbs-devel] [RFC PATCH promxox-backup 2/2] api2: tape/backup: commit pool_writer even on error Dominik Csapak

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal