From: Dietmar Maurer <dietmar@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 1/4] tape restore: split restore_chunk_archive
Date: Mon, 3 May 2021 13:23:34 +0200 [thread overview]
Message-ID: <20210503112337.29879-1-dietmar@proxmox.com> (raw)
Split out a separate function scan_chunk_archive() for catalog restores.
Note: Required, because we need to optimize restore_chunk_archive() to
write datastore in separate threads (else thape drive will stop during restore)
---
src/api2/tape/restore.rs | 90 ++++++++++++++++++++++++++++++----------
1 file changed, 69 insertions(+), 21 deletions(-)
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index b61e99a4..39aa5187 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -150,12 +150,12 @@ impl DataStoreMap {
set
}
- fn get_datastore(&self, source: &str) -> Option<&DataStore> {
+ fn get_datastore(&self, source: &str) -> Option<Arc<DataStore>> {
if let Some(store) = self.map.get(source) {
- return Some(&store);
+ return Some(Arc::clone(store));
}
if let Some(ref store) = self.default {
- return Some(&store);
+ return Some(Arc::clone(store));
}
return None;
@@ -575,10 +575,16 @@ fn restore_archive<'a>(
if datastore.is_some() || target.is_none() {
let checked_chunks = checked_chunks_map
- .entry(datastore.map(|d| d.name()).unwrap_or("_unused_").to_string())
+ .entry(datastore.as_ref().map(|d| d.name()).unwrap_or("_unused_").to_string())
.or_insert(HashSet::new());
- if let Some(chunks) = restore_chunk_archive(worker, reader, datastore, checked_chunks, verbose)? {
+ let chunks = if let Some(datastore) = datastore {
+ restore_chunk_archive(worker, reader, datastore, checked_chunks, verbose)?
+ } else {
+ scan_chunk_archive(worker, reader, verbose)?
+ };
+
+ if let Some(chunks) = chunks {
catalog.start_chunk_archive(
Uuid::from(header.uuid),
current_file_number,
@@ -616,10 +622,56 @@ fn restore_archive<'a>(
Ok(())
}
+// Read chunk archive without restoring data - just record contained chunks
+fn scan_chunk_archive<'a>(
+ worker: &WorkerTask,
+ reader: Box<dyn 'a + TapeRead>,
+ verbose: bool,
+) -> Result<Option<Vec<[u8;32]>>, Error> {
+
+ let mut chunks = Vec::new();
+
+ let mut decoder = ChunkArchiveDecoder::new(reader);
+
+ loop {
+ let digest = match decoder.next_chunk() {
+ Ok(Some((digest, _blob))) => digest,
+ Ok(None) => break,
+ Err(err) => {
+ let reader = decoder.reader();
+
+ // check if this stream is marked incomplete
+ if let Ok(true) = reader.is_incomplete() {
+ return Ok(Some(chunks));
+ }
+
+ // check if this is an aborted stream without end marker
+ if let Ok(false) = reader.has_end_marker() {
+ worker.log("missing stream end marker".to_string());
+ return Ok(None);
+ }
+
+ // else the archive is corrupt
+ return Err(err);
+ }
+ };
+
+ worker.check_abort()?;
+
+ if verbose {
+ task_log!(worker, "Found chunk: {}", proxmox::tools::digest_to_hex(&digest));
+ }
+
+ chunks.push(digest);
+ }
+
+ Ok(Some(chunks))
+}
+
fn restore_chunk_archive<'a>(
worker: &WorkerTask,
reader: Box<dyn 'a + TapeRead>,
- datastore: Option<&DataStore>,
+ datastore: Arc<DataStore>,
checked_chunks: &mut HashSet<[u8;32]>,
verbose: bool,
) -> Result<Option<Vec<[u8;32]>>, Error> {
@@ -653,25 +705,21 @@ fn restore_chunk_archive<'a>(
worker.check_abort()?;
- if let Some(datastore) = datastore {
- let chunk_exists = datastore.cond_touch_chunk(&digest, false)?;
- if !chunk_exists {
- blob.verify_crc()?;
+ let chunk_exists = datastore.cond_touch_chunk(&digest, false)?;
+ if !chunk_exists {
+ blob.verify_crc()?;
- if blob.crypt_mode()? == CryptMode::None {
- blob.decode(None, Some(&digest))?; // verify digest
- }
- if verbose {
- task_log!(worker, "Insert chunk: {}", proxmox::tools::digest_to_hex(&digest));
- }
- datastore.insert_chunk(&blob, &digest)?;
- } else if verbose {
- task_log!(worker, "Found existing chunk: {}", proxmox::tools::digest_to_hex(&digest));
+ if blob.crypt_mode()? == CryptMode::None {
+ blob.decode(None, Some(&digest))?; // verify digest
}
- checked_chunks.insert(digest.clone());
+ if verbose {
+ task_log!(worker, "Insert chunk: {}", proxmox::tools::digest_to_hex(&digest));
+ }
+ datastore.insert_chunk(&blob, &digest)?;
} else if verbose {
- task_log!(worker, "Found chunk: {}", proxmox::tools::digest_to_hex(&digest));
+ task_log!(worker, "Found existing chunk: {}", proxmox::tools::digest_to_hex(&digest));
}
+ checked_chunks.insert(digest.clone());
chunks.push(digest);
}
--
2.20.1
next reply other threads:[~2021-05-03 11:24 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-03 11:23 Dietmar Maurer [this message]
2021-05-03 11:23 ` [pbs-devel] [PATCH proxmox-backup 2/4] tape restore: write datastore in separate thread Dietmar Maurer
2021-05-03 11:23 ` [pbs-devel] [PATCH proxmox-backup 3/4] tape restore: add restore speed to logs Dietmar Maurer
2021-05-03 11:23 ` [pbs-devel] [PATCH proxmox-backup 4/4] tape restore: do not verify restored files Dietmar Maurer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210503112337.29879-1-dietmar@proxmox.com \
--to=dietmar@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox