From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 883E41FF16B for ; Mon, 16 Sep 2024 16:56:28 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 515D7A2F4; Mon, 16 Sep 2024 16:56:34 +0200 (CEST) From: Gabriel Goller To: pbs-devel@lists.proxmox.com Date: Mon, 16 Sep 2024 16:56:27 +0200 Message-Id: <20240916145627.515861-2-g.goller@proxmox.com> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20240916145627.515861-1-g.goller@proxmox.com> References: <20240916145627.515861-1-g.goller@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.044 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 2/2] api: parallelize smartctl checks X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: multipart/mixed; boundary="===============5227709903943140991==" Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" --===============5227709903943140991== Content-Type: text/plain; charset=y Content-Transfer-Encoding: 8bit To improve the performance of the smartctl checks, especially when a lot of disks are used, parallelize the checks using the `ParallelHandler`. Signed-off-by: Gabriel Goller --- Benchmark: Benchmark 1: proxmox-backup-manager disk list (parallel) Time (mean ± σ): 75.8 ms ± 5.7 ms [User: 0.8 ms, System: 0.5 ms] Range (min … max): 63.4 ms … 95.3 ms 100 runs Benchmark 2: proxmox-backup-manager disk list (sequential) Time (mean ± σ): 189.4 ms ± 6.7 ms [User: 0.7 ms, System: 1.0 ms] Range (min … max): 178.8 ms … 223.0 ms 100 runs Summary 'proxmox-backup-manager disk list (parallel)' ran 2.50 ± 0.21 times faster than 'proxmox-backup-manager disk list (sequential)' src/tools/disks/mod.rs | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/tools/disks/mod.rs b/src/tools/disks/mod.rs index 04f62b818238..09a3003a9333 100644 --- a/src/tools/disks/mod.rs +++ b/src/tools/disks/mod.rs @@ -15,13 +15,15 @@ use once_cell::sync::OnceCell; use ::serde::{Deserialize, Serialize}; use proxmox_lang::{io_bail, io_format_err}; +use proxmox_log::info; use proxmox_schema::api; use proxmox_sys::linux::procfs::{mountinfo::Device, MountInfo}; use pbs_api_types::{BLOCKDEVICE_DISK_AND_PARTITION_NAME_REGEX, BLOCKDEVICE_NAME_REGEX}; +use crate::tools::parallel_handler::ParallelHandler; + mod zfs; -use tracing::info; pub use zfs::*; mod zpool_status; pub use zpool_status::*; @@ -1047,16 +1049,6 @@ fn get_disks( usage = DiskUsageType::DeviceMapper; } - let mut status = SmartStatus::Unknown; - let mut wearout = None; - - if !no_smart { - if let Ok(smart) = get_smart_data(&disk, false) { - status = smart.status; - wearout = smart.wearout; - } - } - let info = DiskUsageInfo { name: name.clone(), vendor, @@ -1067,8 +1059,8 @@ fn get_disks( size, wwn, disk_type, - status, - wearout, + status: SmartStatus::Unknown, + wearout: None, used: usage, gpt: disk.has_gpt(), rpm: disk.ata_rotation_rate_rpm(), @@ -1077,6 +1069,30 @@ fn get_disks( result.insert(name, info); } + if !no_smart { + let (tx, rx) = crossbeam_channel::bounded(result.len()); + + let parallel_handler = + ParallelHandler::new("smartctl data", 4, move |device: (String, String)| { + let smart_data = get_smart_data(Path::new(&device.1), false)?; + tx.send((device.0, smart_data))?; + Ok(()) + }); + + for (name, path) in device_paths.into_iter() { + if let Some(p) = path { + parallel_handler.send((name, p))?; + } + } + + parallel_handler.complete()?; + while let Ok(msg) = rx.recv() { + if let Some(value) = result.get_mut(&msg.0) { + value.wearout = msg.1.wearout; + value.status = msg.1.status; + } + } + } Ok(result) } -- 2.39.5 --===============5227709903943140991== Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: inline _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel --===============5227709903943140991==--