From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id D771E1FF165 for ; Thu, 3 Jul 2025 15:19:57 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0D53E14AF7; Thu, 3 Jul 2025 15:20:17 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 3 Jul 2025 15:18:10 +0200 Message-ID: <20250703131837.786811-23-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250703131837.786811-1-c.ebner@proxmox.com> References: <20250703131837.786811-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.040 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v5 19/46] api: backup: conditionally upload indices to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" If the datastore is backed by an S3 compatible object store, upload the dynamic or fixed index files to the object store after closing them. The local index files are kept in the local caching datastore to allow for fast and efficient content lookups, avoiding expensive (as in monetary cost and IO latency) requests. Signed-off-by: Christian Ebner --- src/api2/backup/environment.rs | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index dbf0ebccf..0cc6cb8f7 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -2,6 +2,7 @@ use anyhow::{bail, format_err, Context, Error}; use pbs_config::BackupLockGuard; use std::collections::HashMap; +use std::io::Read; use std::sync::{Arc, Mutex}; use tracing::info; @@ -17,6 +18,7 @@ use pbs_datastore::backup_info::{BackupDir, BackupInfo}; use pbs_datastore::dynamic_index::DynamicIndexWriter; use pbs_datastore::fixed_index::FixedIndexWriter; use pbs_datastore::{DataBlob, DataStore, DatastoreBackend}; +use pbs_s3_client::S3Client; use proxmox_rest_server::{formatter::*, WorkerTask}; use crate::backup::VerifyWorker; @@ -479,6 +481,13 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload dynamic index to s3 backend")?; + self.log(format!("Uploaded index file to s3 backend: {}", data.name)) + } + self.log_upload_stat( &data.name, &csum, @@ -553,6 +562,16 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload fixed index to s3 backend")?; + self.log(format!( + "Uploaded fixed index file to object store: {}", + data.name + )) + } + self.log_upload_stat( &data.name, &expected_csum, @@ -754,6 +773,28 @@ impl BackupEnvironment { Ok(()) } + + fn s3_upload_index(&self, s3_client: &S3Client, name: &str) -> Result<(), Error> { + let mut object_key = self.backup_dir.relative_path(); + object_key.push(name); + let object_key = object_key + .as_os_str() + .to_str() + .ok_or_else(|| format_err!("invalid file name"))?; + + let mut full_path = self.datastore.base_path(); + full_path.push(object_key); + let mut file = std::fs::File::open(&full_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + let data = hyper::body::Bytes::from(buffer); + proxmox_async::runtime::block_on(s3_client.upload_with_retry( + object_key.into(), + data, + true, + ))?; + Ok(()) + } } impl RpcEnvironment for BackupEnvironment { -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel