From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id D96D91FF185 for ; Mon, 23 Jun 2025 11:41:09 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 09A68CF44; Mon, 23 Jun 2025 11:41:35 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 23 Jun 2025 11:40:40 +0200 Message-ID: <20250623094106.299251-23-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250623094106.299251-1-c.ebner@proxmox.com> References: <20250623094106.299251-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.038 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v4 19/45] api: backup: conditionally upload indices to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" If the datastore is backed by an S3 compatible object store, upload the dynamic or fixed index files to the object store after closing them. The local index files are kept in the local caching datastore to allow for fast and efficient content lookups, avoiding expensive (as in monetary cost and IO latency) requests. Signed-off-by: Christian Ebner --- src/api2/backup/environment.rs | 65 ++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index bdaa27350..ec4d2d457 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -2,6 +2,7 @@ use anyhow::{bail, format_err, Context, Error}; use pbs_config::BackupLockGuard; use std::collections::HashMap; +use std::io::Read; use std::sync::{Arc, Mutex}; use tracing::info; @@ -480,6 +481,38 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + let mut object_key = self.backup_dir.relative_path(); + object_key.push(&data.name); + let object_key = object_key + .as_os_str() + .to_str() + .ok_or_else(|| format_err!("invalid file name"))?; + + let mut full_path = self.datastore.base_path(); + full_path.push(object_key); + let mut file = std::fs::File::open(&full_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + let data = Body::from(buffer); + match proxmox_async::runtime::block_on(s3_client.put_object(object_key.into(), data))? { + PutObjectResponse::PreconditionFailed => { + self.log(format!( + "Upload of dynamic index failed, object key {object_key} already present" + )); + bail!("Upload of dynamic index failed"); + } + PutObjectResponse::NeedsRetry => { + self.log("Upload of dynamic index failed, reupload required"); + bail!("concurrent operation, reupload required"); + } + PutObjectResponse::Success(_content) => { + self.log(format!("Uploaded index file to object store: {object_key}")) + } + } + } + self.log_upload_stat( &data.name, &csum, @@ -554,6 +587,38 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + let mut object_key = self.backup_dir.relative_path(); + object_key.push(&data.name); + let object_key = object_key + .as_os_str() + .to_str() + .ok_or_else(|| format_err!("invalid file name"))?; + + let mut full_path = self.datastore.base_path(); + full_path.push(object_key); + let mut file = std::fs::File::open(&full_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + let data = Body::from(buffer); + match proxmox_async::runtime::block_on(s3_client.put_object(object_key.into(), data))? { + PutObjectResponse::PreconditionFailed => { + self.log(format!( + "Upload of fixed index failed, object {object_key} already present." + )); + bail!("upload of fixed index failed"); + } + PutObjectResponse::NeedsRetry => { + self.log("Upload of fixed index failed, reupload required."); + bail!("concurrent operation, reupload required"); + } + PutObjectResponse::Success(_content) => { + self.log(format!("Uploaded index file to object store: {object_key}")) + } + } + } + self.log_upload_stat( &data.name, &expected_csum, -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel