From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 735D01FF0E4 for ; Tue, 11 Aug 2026 11:37:58 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 31F622155C; Tue, 11 Aug 2026 11:37:58 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20251104; t=1786441068; x=1787045868; darn=lists.proxmox.com; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to:content-type; bh=1nB8ttIwmvf+3XiPyRdcvFXSCxI++LSr4WG7kbWrbV8=; b=n4ecwnRIjKivrjolKm+aLTyO7QHUFJU1qTeqeXBVmvBosUeGft3wbg/rAVTqZ0Q17Q uqXP2bnmS38qdRhZ23OSaA449pTxRttUy3MjfggkGqAMAtUuAuE1opC7ScntMn6TPstg MOwQcBI+l8TPskXki88BvckON1HHEtmxc45WSXuRXLG1JEVHoxjVZ6nLGgiR9iyAPYSt 63bizbjScoXmFydF6gFYxxdCU71DltgA9QQkHob15ALnDS7LAw9dFnFlgg1PAxjUNIVC y+PPalMxXUMXwbtUg3Z0+ulAF6ddl6wAGXDAo9jh2l9EruGydvFLbdd8rimifyoh5WSo msFg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1786441068; x=1787045868; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-gg:x-gm-message-state:from :to:cc:subject:date:message-id:reply-to:content-type; bh=1nB8ttIwmvf+3XiPyRdcvFXSCxI++LSr4WG7kbWrbV8=; b=MCnlKcDigUN1oXyN75S9df2hIoZ92C+rqNMCg4rDeVxw9JGM/jlXip46dzD1YtAd2h epU4T79OyslYSoz+rVh7YiDjLL/H4JwLppGICKXvllC5QC9ImKjrWcAZFWHOISapwKYO ksLWrd688hfRrCmY5w//vZOM+Y6L2j11tkm2Bpe+2ceugOBjhTUwOKHaR63ITX62DUv8 qO04LVV+2RyWj3BZA1fB8dBlPKq7EBNyiWCnYk+SCCokwKBA/m10d7kCIZtPWhYOM0mh UnjeZwwPkmCFQ0L4h+k0bA6+q9XgjM+CI+n2I0VgzHVn7wP+f60CEjsZP40/j9B/UI2C dm1w== X-Gm-Message-State: AOJu0Yy8Oj1j8r2447GHF9dbQ7haYhvJNxw3rqU4+V0fcFuU65QubzaA jqSWdtGEAHhCLivqlp/xYa7qXWO5MLqb/VkpNnw9pRf8cWq6rRQCWoPIehwxLw== X-Gm-Gg: AR+sD11WvspkfWWticwFAexPxmfU73oakHtyTQMY5ZINWLLNcvtGzAXF7uBDsYTu77p E2trAjDQLU8D/LBEeuYClz4Cenb2v+iN6RbnagNlhHc1LhK6+tUBNznDBIRZl9U+Cntu8kqJH/k ZE1Bzv8nIRnwMbe4eJjgXpK53vaOq3eHyMUa6/Alq4UB/GULSPEzhY59ydJDttGIUQmZ50cyxFz QlHuOIXA1MiH+p1DwrZION2YDPprmmAhkJFzq2jqMabLJ7HdRmcIg0zmMlKnPOX3uv3+XWZ/9Nn ltX1Hqh+k6ma2N2595Z4sZGY4k0RWsYFvfQcGCh+IjKXpDhUuZmdiMsDuX/88lNhlurUyYAeJBR bevd6CIg6F81OTLrPkDABmAeQi0x2Fj0dhsAibMb2VbSaMTats65eXN2x3XQsf6HRE0imliW7Lk mHCKWcIvCaevAm6EBDEgwCPb99gf0pwME6jYs+JKN4T3SwdDIjC61RlmpeV0UYK0G91lTTnf6im jFKT5SZ5Twd7ZkR24a9I2x/3tnHL9sngKp+ZYJ7WSHL9y1l//iXZPWauWM9jQOEe6IVhw== X-Received: by 2002:a05:600c:c172:b0:495:4689:1e98 with SMTP id 5b1f17b1804b1-49978463c3bmr33854445e9.10.1786441067814; Tue, 11 Aug 2026 02:37:47 -0700 (PDT) From: Enrico Plantulli To: pbs-devel@lists.proxmox.com Subject: [PATCH v2 proxmox 1/3] pbs-api-types: add gc chunk metadata prefetch tuning options Date: Tue, 11 Aug 2026 11:37:19 +0200 Message-ID: <20260811093722.735290-2-plantulli@gmail.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260811093722.735290-1-plantulli@gmail.com> References: <20260810050201.347124-1-plantulli@gmail.com> <20260811093722.735290-1-plantulli@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.300 Adjusted score from AWL reputation of From: address DKIM_SIGNED 0.1 Message has a DKIM or DK signature, not necessarily valid DKIM_VALID -0.1 Message has at least one valid DKIM or DK signature DKIM_VALID_AU -0.1 Message has a valid DKIM or DK signature from author's domain DKIM_VALID_EF -0.1 Message has a valid DKIM or DK signature from envelope-from domain DMARC_PASS -0.1 DMARC pass policy FREEMAIL_FROM 0.001 Sender email is commonly abused enduser mail provider RCVD_IN_DNSWL_NONE -0.0001 Sender listed at https://www.dnswl.org/, no trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: MLYHLQLPUK2S2D6IXLE7CFTHQEEFK26J X-Message-ID-Hash: MLYHLQLPUK2S2D6IXLE7CFTHQEEFK26J X-MailFrom: plantulli@gmail.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Enrico Plantulli X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add an opt-in option to defer and batch chunk access time updates in garbage collection phase 1, plus a separate option for the batch size to ease benchmarking on different storage backends. Batched updates are flushed sorted by digest, so the chunk directories involved can be warmed via readdir right before their chunks are touched. Keeping the switch opt-in follows the review of the first version of this series: storage backends that do not prefetch inode metadata on readdir, such as some network attached storages, may see no benefit or a regression. Signed-off-by: Enrico Plantulli --- pbs-api-types/src/datastore.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs index 93ccaf0..79ccff9 100644 --- a/pbs-api-types/src/datastore.rs +++ b/pbs-api-types/src/datastore.rs @@ -256,6 +256,14 @@ pub const GC_CACHE_CAPACITY_SCHEMA: Schema = .default(1024 * 1024) .schema(); +pub const GC_PREFETCH_BATCH_SIZE_SCHEMA: Schema = IntegerSchema::new( + "Batch size for deferred chunk access time updates in garbage collection phase 1", +) +.minimum(1024) +.maximum(16 * 1024 * 1024) +.default(1024 * 1024) +.schema(); + #[api( properties: { "chunk-order": { @@ -278,6 +286,18 @@ pub const GC_CACHE_CAPACITY_SCHEMA: Schema = schema: GC_CACHE_CAPACITY_SCHEMA, optional: true, }, + "gc-chunk-metadata-prefetch": { + description: + "Defer and batch chunk access time updates in garbage collection phase 1, \ + warming each chunk directory via readdir right before its chunks are touched", + optional: true, + default: false, + type: bool, + }, + "gc-prefetch-batch-size": { + schema: GC_PREFETCH_BATCH_SIZE_SCHEMA, + optional: true, + }, "default-verification-workers": { schema: VERIFY_JOB_VERIFY_THREADS_SCHEMA, optional: true, @@ -304,6 +324,10 @@ pub struct DatastoreTuning { #[serde(skip_serializing_if = "Option::is_none")] pub gc_cache_capacity: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub gc_chunk_metadata_prefetch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub gc_prefetch_batch_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub default_verification_workers: Option, #[serde(skip_serializing_if = "Option::is_none")] pub default_verification_readers: Option, base-commit: e3e3ff11b9b92fe1ace89b84c1e15c150e2db660 -- 2.47.3