From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id ECE2320EC91 for ; Tue, 30 Apr 2024 11:39:33 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id C1C7930081; Tue, 30 Apr 2024 11:39:42 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Tue, 30 Apr 2024 11:39:38 +0200 Message-Id: <20240430093939.1318786-1-d.csapak@proxmox.com> X-Mailer: git-send-email 2.39.2 MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.016 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [datastore.rs, verify.rs] Subject: [pbs-devel] [PATCH proxmox-backup 1/2] datastore: add tuning options for the number of reading threads X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" adds a new 'read-threads' tuning options to datastores that control how many threads are used for verification operations depending on the underlying storage and the used cpu, this can make a big difference in verification time. the default remains at the current 4 threads. Signed-off-by: Dominik Csapak --- pbs-api-types/src/datastore.rs | 9 +++++++++ pbs-datastore/src/datastore.rs | 14 ++++++++++++++ src/backup/verify.rs | 2 +- www/Utils.js | 5 +++++ www/datastore/OptionView.js | 8 ++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs index 31767417a..2ad2ae063 100644 --- a/pbs-api-types/src/datastore.rs +++ b/pbs-api-types/src/datastore.rs @@ -209,6 +209,13 @@ pub enum DatastoreFSyncLevel { type: ChunkOrder, optional: true, }, + "read-threads": { + description: "Controls how many threads are used for reading from the datastore for verification.", + type: usize, + optional: true, + minimum: 1, + default: 4, + }, }, )] #[derive(Serialize, Deserialize, Default)] @@ -220,6 +227,8 @@ pub struct DatastoreTuning { pub chunk_order: Option, #[serde(skip_serializing_if = "Option::is_none")] pub sync_level: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub read_threads: Option, } pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore tuning options") diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index f95da7615..2c23bd1f1 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -14,6 +14,7 @@ use proxmox_schema::ApiType; use proxmox_sys::error::SysError; use proxmox_sys::fs::{file_read_optional_string, replace_file, CreateOptions}; use proxmox_sys::fs::{lock_dir_noblock, DirLockGuard}; +use proxmox_sys::linux::procfs::read_proc_stat; use proxmox_sys::process_locker::ProcessLockSharedGuard; use proxmox_sys::WorkerTaskContext; use proxmox_sys::{task_log, task_warn}; @@ -61,6 +62,7 @@ pub struct DataStoreImpl { chunk_order: ChunkOrder, last_digest: Option<[u8; 32]>, sync_level: DatastoreFSyncLevel, + read_threads: usize, } impl DataStoreImpl { @@ -75,6 +77,7 @@ impl DataStoreImpl { chunk_order: Default::default(), last_digest: None, sync_level: Default::default(), + read_threads: 0, }) } } @@ -313,6 +316,7 @@ impl DataStore { chunk_order: tuning.chunk_order.unwrap_or_default(), last_digest, sync_level: tuning.sync_level.unwrap_or_default(), + read_threads: tuning.read_threads.unwrap_or(4), }) } @@ -1377,6 +1381,16 @@ impl DataStore { Ok(()) } + /// returns the number of read thread that should be used for operations + /// limits to the number of available cores (if reading from /proc/stat succeeds) + pub fn get_read_threads(&self) -> usize { + // if we cannot get the real cpu count, don't limit us + let core_count = read_proc_stat() + .map(|proc| proc.cpu_count as usize) + .unwrap_or(usize::max_value()); + self.inner.read_threads.clamp(1, core_count) + } + /// Destroy a datastore. This requires that there are no active operations on the datastore. /// /// This is a synchronous operation and should be run in a worker-thread. diff --git a/src/backup/verify.rs b/src/backup/verify.rs index c972e5328..4537dbdc9 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -125,7 +125,7 @@ fn verify_index_chunks( let decoder_pool = ParallelHandler::new( "verify chunk decoder", - 4, + datastore2.get_read_threads(), move |(chunk, digest, size): (DataBlob, [u8; 32], u64)| { let chunk_crypt_mode = match chunk.crypt_mode() { Err(err) => { diff --git a/www/Utils.js b/www/Utils.js index 1d7351a32..8fd102486 100644 --- a/www/Utils.js +++ b/www/Utils.js @@ -790,6 +790,11 @@ Ext.define('PBS.Utils', { sync = PBS.Utils.tuningOptions['sync-level'][sync ?? '__default__']; options.push(`${gettext('Sync Level')}: ${sync}`); + let readThreads = tuning['read-threads']; + delete tuning['read-threads']; + readThreads ??= Proxmox.Utils.defaultText + ` (4)`; + options.push(`${gettext('Read Threads')}: ${readThreads}`); + for (const [k, v] of Object.entries(tuning)) { options.push(`${k}: ${v}`); } diff --git a/www/datastore/OptionView.js b/www/datastore/OptionView.js index e1f38af6f..ee90f1dca 100644 --- a/www/datastore/OptionView.js +++ b/www/datastore/OptionView.js @@ -271,6 +271,14 @@ Ext.define('PBS.Datastore.Options', { deleteEmpty: true, value: '__default__', }, + { + xtype: 'proxmoxintegerfield', + name: 'read-threads', + fieldLabel: gettext('Read Threads'), + min: 0, + emptyText: '4', + deleteEmpty: true, + }, ], }, }, -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel