From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 14B481FF0AE for ; Tue, 15 Sep 2026 18:01:01 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id B160521600; Tue, 15 Sep 2026 18:00:58 +0200 (CEST) From: Shan Shaji To: pbs-devel@lists.proxmox.com Subject: [RFC PATCH proxmox-backup 2/4] fix #7908: datastore: create zero-byte markers when re-using datastore Date: Tue, 15 Sep 2026 17:59:33 +0200 Message-ID: <20260915155935.135460-3-s.shaji@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260915155935.135460-1-s.shaji@proxmox.com> References: <20260915155935.135460-1-s.shaji@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1789488040515 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.462 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: VUYDON62P27PWWCZ5MJANTUUWBOMJGQ2 X-Message-ID-Hash: VUYDON62P27PWWCZ5MJANTUUWBOMJGQ2 X-MailFrom: s.shaji@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Previously, recreating an S3-backed datastore with the --reuse-datastore flag fetched only the metadata files into the local cache, without creating chunk markers. Garbage collection (GC) had to be run to recreate markers for chunks referenced by the backup indexes. In order to fix this, create local zero-byte chunk markers by listing the chunk objects in S3 when reusing a datastore. Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=7908 Signed-off-by: Shan Shaji --- pbs-datastore/src/datastore.rs | 35 +++++++++++++++++++++++++++++++++- src/api2/admin/datastore.rs | 10 +++++++--- src/api2/config/datastore.rs | 6 +++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index f3eab8a28..d00a0ef1c 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -3171,9 +3171,38 @@ impl DataStore { *OLD_LOCKING } + fn create_empty_s3_chunk_markers( + &self, + worker: &dyn WorkerTaskContext, + s3_client: &S3Client, + ) -> Result<(), Error> { + let _guard = self.inner.chunk_store.mutex().lock().unwrap(); + + self.paginate_s3_chunk_objects(worker, s3_client, |object_key, _| -> Result<(), Error> { + let (_, digest, _) = match digest_from_object_key(&object_key) { + Some(result) => result, + None => return Ok(()), + }; + + unsafe { + self.inner + .chunk_store + .replace_chunk_with_marker_or_create_marker(&digest)?; + } + + Ok(()) + })?; + + Ok(()) + } + /// Fetch contents from S3 object store, clear and replace the local cache store contents. /// Returns with error for non-S3 datastore backends. - pub async fn s3_refresh(self: &Arc) -> Result<(), Error> { + pub async fn s3_refresh( + self: &Arc, + rebuild_chunk_markers: bool, + worker: &dyn WorkerTaskContext, + ) -> Result<(), Error> { match self.backend()? { DatastoreBackend::Filesystem => bail!("store '{}' not backed by S3", self.name()), DatastoreBackend::S3(s3_client) => { @@ -3190,6 +3219,10 @@ impl DataStore { let _ = std::fs::remove_dir_all(&tmp_base); return Err(err); } + + if rebuild_chunk_markers { + self.create_empty_s3_chunk_markers(worker, &s3_client)?; + } } } Ok(()) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index bc2b2436e..34907fa0a 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -2867,7 +2867,7 @@ pub fn s3_refresh(store: String, rpcenv: &mut dyn RpcEnvironment) -> Result Result Result<(), Error> { +pub(crate) fn do_s3_refresh( + store: &str, + worker: &dyn WorkerTaskContext, + rebuild_chunk_markers: bool, +) -> Result<(), Error> { let datastore = DataStore::lookup_datastore(lookup_with(store, Operation::Lookup))?; run_maintenance_locked(store, MaintenanceType::S3Refresh, worker, || { - proxmox_async::runtime::block_on(datastore.s3_refresh()) + proxmox_async::runtime::block_on(datastore.s3_refresh(rebuild_chunk_markers, worker)) }) } diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs index e7028480c..e82e8969a 100644 --- a/src/api2/config/datastore.rs +++ b/src/api2/config/datastore.rs @@ -366,7 +366,11 @@ pub fn create_datastore( } if reuse_datastore && backend == DatastoreBackendType::S3 { - crate::api2::admin::datastore::do_s3_refresh(&store_name, &worker)?; + crate::api2::admin::datastore::do_s3_refresh( + &store_name, + &worker, + reuse_datastore, + )?; } Ok(()) }, -- 2.47.3