From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 7837F7902E for ; Mon, 3 May 2021 13:24:16 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 722D31CD61 for ; Mon, 3 May 2021 13:23:46 +0200 (CEST) Received: from elsa.proxmox.com (unknown [94.136.29.99]) by firstgate.proxmox.com (Proxmox) with ESMTP id DF4271CD3B for ; Mon, 3 May 2021 13:23:44 +0200 (CEST) Received: by elsa.proxmox.com (Postfix, from userid 0) id B2013AEB2C0; Mon, 3 May 2021 13:23:38 +0200 (CEST) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Mon, 3 May 2021 13:23:34 +0200 Message-Id: <20210503112337.29879-1-dietmar@proxmox.com> X-Mailer: git-send-email 2.20.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.360 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 1/4] tape restore: split restore_chunk_archive X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 03 May 2021 11:24:16 -0000 Split out a separate function scan_chunk_archive() for catalog restores. Note: Required, because we need to optimize restore_chunk_archive() to write datastore in separate threads (else thape drive will stop during restore) --- src/api2/tape/restore.rs | 90 ++++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs index b61e99a4..39aa5187 100644 --- a/src/api2/tape/restore.rs +++ b/src/api2/tape/restore.rs @@ -150,12 +150,12 @@ impl DataStoreMap { set } - fn get_datastore(&self, source: &str) -> Option<&DataStore> { + fn get_datastore(&self, source: &str) -> Option> { if let Some(store) = self.map.get(source) { - return Some(&store); + return Some(Arc::clone(store)); } if let Some(ref store) = self.default { - return Some(&store); + return Some(Arc::clone(store)); } return None; @@ -575,10 +575,16 @@ fn restore_archive<'a>( if datastore.is_some() || target.is_none() { let checked_chunks = checked_chunks_map - .entry(datastore.map(|d| d.name()).unwrap_or("_unused_").to_string()) + .entry(datastore.as_ref().map(|d| d.name()).unwrap_or("_unused_").to_string()) .or_insert(HashSet::new()); - if let Some(chunks) = restore_chunk_archive(worker, reader, datastore, checked_chunks, verbose)? { + let chunks = if let Some(datastore) = datastore { + restore_chunk_archive(worker, reader, datastore, checked_chunks, verbose)? + } else { + scan_chunk_archive(worker, reader, verbose)? + }; + + if let Some(chunks) = chunks { catalog.start_chunk_archive( Uuid::from(header.uuid), current_file_number, @@ -616,10 +622,56 @@ fn restore_archive<'a>( Ok(()) } +// Read chunk archive without restoring data - just record contained chunks +fn scan_chunk_archive<'a>( + worker: &WorkerTask, + reader: Box, + verbose: bool, +) -> Result>, Error> { + + let mut chunks = Vec::new(); + + let mut decoder = ChunkArchiveDecoder::new(reader); + + loop { + let digest = match decoder.next_chunk() { + Ok(Some((digest, _blob))) => digest, + Ok(None) => break, + Err(err) => { + let reader = decoder.reader(); + + // check if this stream is marked incomplete + if let Ok(true) = reader.is_incomplete() { + return Ok(Some(chunks)); + } + + // check if this is an aborted stream without end marker + if let Ok(false) = reader.has_end_marker() { + worker.log("missing stream end marker".to_string()); + return Ok(None); + } + + // else the archive is corrupt + return Err(err); + } + }; + + worker.check_abort()?; + + if verbose { + task_log!(worker, "Found chunk: {}", proxmox::tools::digest_to_hex(&digest)); + } + + chunks.push(digest); + } + + Ok(Some(chunks)) +} + fn restore_chunk_archive<'a>( worker: &WorkerTask, reader: Box, - datastore: Option<&DataStore>, + datastore: Arc, checked_chunks: &mut HashSet<[u8;32]>, verbose: bool, ) -> Result>, Error> { @@ -653,25 +705,21 @@ fn restore_chunk_archive<'a>( worker.check_abort()?; - if let Some(datastore) = datastore { - let chunk_exists = datastore.cond_touch_chunk(&digest, false)?; - if !chunk_exists { - blob.verify_crc()?; + let chunk_exists = datastore.cond_touch_chunk(&digest, false)?; + if !chunk_exists { + blob.verify_crc()?; - if blob.crypt_mode()? == CryptMode::None { - blob.decode(None, Some(&digest))?; // verify digest - } - if verbose { - task_log!(worker, "Insert chunk: {}", proxmox::tools::digest_to_hex(&digest)); - } - datastore.insert_chunk(&blob, &digest)?; - } else if verbose { - task_log!(worker, "Found existing chunk: {}", proxmox::tools::digest_to_hex(&digest)); + if blob.crypt_mode()? == CryptMode::None { + blob.decode(None, Some(&digest))?; // verify digest } - checked_chunks.insert(digest.clone()); + if verbose { + task_log!(worker, "Insert chunk: {}", proxmox::tools::digest_to_hex(&digest)); + } + datastore.insert_chunk(&blob, &digest)?; } else if verbose { - task_log!(worker, "Found chunk: {}", proxmox::tools::digest_to_hex(&digest)); + task_log!(worker, "Found existing chunk: {}", proxmox::tools::digest_to_hex(&digest)); } + checked_chunks.insert(digest.clone()); chunks.push(digest); } -- 2.20.1