From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 734911FF185 for ; Mon, 21 Jul 2025 18:44:26 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id F0B7115D7D; Mon, 21 Jul 2025 18:45:30 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 21 Jul 2025 18:44:31 +0200 Message-ID: <20250721164507.1045869-14-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250721164507.1045869-1-c.ebner@proxmox.com> References: <20250721164507.1045869-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1753116306516 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.045 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v10 10/46] api: backup: conditionally upload indices to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" If the datastore is backed by an S3 compatible object store, upload the dynamic or fixed index files to the object store after closing them. The local index files are kept in the local caching datastore to allow for fast and efficient content lookups, avoiding expensive (as in monetary cost and IO latency) requests. Signed-off-by: Christian Ebner --- changes since version 9: - no changes src/api2/backup/environment.rs | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index a5683b845..b67145fa6 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -18,6 +18,7 @@ use pbs_datastore::dynamic_index::DynamicIndexWriter; use pbs_datastore::fixed_index::FixedIndexWriter; use pbs_datastore::{DataBlob, DataStore, DatastoreBackend}; use proxmox_rest_server::{formatter::*, WorkerTask}; +use proxmox_s3_client::S3Client; use crate::backup::VerifyWorker; @@ -479,6 +480,16 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload dynamic index to s3 backend")?; + self.log(format!( + "Uploaded dynamic index file to s3 backend: {}", + data.name + )) + } + self.log_upload_stat( &data.name, &csum, @@ -553,6 +564,16 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload fixed index to s3 backend")?; + self.log(format!( + "Uploaded fixed index file to object store: {}", + data.name + )) + } + self.log_upload_stat( &data.name, &expected_csum, @@ -751,6 +772,21 @@ impl BackupEnvironment { Ok(()) } + + fn s3_upload_index(&self, s3_client: &S3Client, name: &str) -> Result<(), Error> { + let object_key = + pbs_datastore::s3::object_key_from_path(&self.backup_dir.relative_path(), name) + .context("invalid index file object key")?; + + let mut full_path = self.backup_dir.full_path(); + full_path.push(name); + let data = std::fs::read(&full_path).context("failed to read index contents")?; + let contents = hyper::body::Bytes::from(data); + proxmox_async::runtime::block_on( + s3_client.upload_replace_with_retry(object_key, contents), + )?; + Ok(()) + } } impl RpcEnvironment for BackupEnvironment { -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel