From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 2E570790A4 for ; Mon, 3 May 2021 13:23:46 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 253371CD5F for ; Mon, 3 May 2021 13:23:46 +0200 (CEST) Received: from elsa.proxmox.com (unknown [94.136.29.99]) by firstgate.proxmox.com (Proxmox) with ESMTP id DB9B61CD3A for ; Mon, 3 May 2021 13:23:44 +0200 (CEST) Received: by elsa.proxmox.com (Postfix, from userid 0) id B4BDCAEB06B; Mon, 3 May 2021 13:23:38 +0200 (CEST) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Mon, 3 May 2021 13:23:35 +0200 Message-Id: <20210503112337.29879-2-dietmar@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210503112337.29879-1-dietmar@proxmox.com> References: <20210503112337.29879-1-dietmar@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.369 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 2/4] tape restore: write datastore in separate thread X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 03 May 2021 11:23:46 -0000 --- src/api2/tape/restore.rs | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs index 39aa5187..97fd822f 100644 --- a/src/api2/tape/restore.rs +++ b/src/api2/tape/restore.rs @@ -32,7 +32,7 @@ use crate::{ task_log, task_warn, task::TaskState, - tools::compute_file_csum, + tools::{compute_file_csum, ParallelHandler}, api2::types::{ DATASTORE_MAP_ARRAY_SCHEMA, DATASTORE_MAP_LIST_SCHEMA, @@ -680,6 +680,24 @@ fn restore_chunk_archive<'a>( let mut decoder = ChunkArchiveDecoder::new(reader); + let datastore2 = datastore.clone(); + let writer_pool = ParallelHandler::new( + "tape restore chunk writer", + 4, + move |(chunk, digest): (DataBlob, [u8; 32])| { + // println!("verify and write {}", proxmox::tools::digest_to_hex(&digest)); + chunk.verify_crc()?; + if chunk.crypt_mode()? == CryptMode::None { + chunk.decode(None, Some(&digest))?; // verify digest + } + + datastore2.insert_chunk(&chunk, &digest)?; + Ok(()) + }, + ); + + let verify_and_write_channel = writer_pool.channel(); + loop { let (digest, blob) = match decoder.next_chunk() { Ok(Some((digest, blob))) => (digest, blob), @@ -707,22 +725,21 @@ fn restore_chunk_archive<'a>( let chunk_exists = datastore.cond_touch_chunk(&digest, false)?; if !chunk_exists { - blob.verify_crc()?; - - if blob.crypt_mode()? == CryptMode::None { - blob.decode(None, Some(&digest))?; // verify digest - } - if verbose { + if verbose { task_log!(worker, "Insert chunk: {}", proxmox::tools::digest_to_hex(&digest)); } - datastore.insert_chunk(&blob, &digest)?; - } else if verbose { + verify_and_write_channel.send((blob, digest.clone()))?; + } else if verbose { task_log!(worker, "Found existing chunk: {}", proxmox::tools::digest_to_hex(&digest)); } checked_chunks.insert(digest.clone()); chunks.push(digest); } + drop(verify_and_write_channel); + + writer_pool.complete()?; + Ok(Some(chunks)) } -- 2.20.1