From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 0F4911FF170 for ; Thu, 10 Jul 2025 19:07:46 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id F34B33966A; Thu, 10 Jul 2025 19:08:24 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 10 Jul 2025 19:07:00 +0200 Message-ID: <20250710170728.102829-20-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250710170728.102829-1-c.ebner@proxmox.com> References: <20250710170728.102829-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.043 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v7 10/38] api: backup: conditionally upload indices to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" If the datastore is backed by an S3 compatible object store, upload the dynamic or fixed index files to the object store after closing them. The local index files are kept in the local caching datastore to allow for fast and efficient content lookups, avoiding expensive (as in monetary cost and IO latency) requests. Signed-off-by: Christian Ebner --- changes since version 6: - use s3 object key generation helper, thereby fixing missing content prefix regression. src/api2/backup/environment.rs | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index 3d4677975..8131ee9b2 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -2,6 +2,7 @@ use anyhow::{bail, format_err, Context, Error}; use pbs_config::BackupLockGuard; use std::collections::HashMap; +use std::io::Read; use std::sync::{Arc, Mutex}; use tracing::info; @@ -18,6 +19,7 @@ use pbs_datastore::dynamic_index::DynamicIndexWriter; use pbs_datastore::fixed_index::FixedIndexWriter; use pbs_datastore::{DataBlob, DataStore, DatastoreBackend}; use proxmox_rest_server::{formatter::*, WorkerTask}; +use proxmox_s3_client::S3Client; use crate::backup::VerifyWorker; @@ -479,6 +481,13 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload dynamic index to s3 backend")?; + self.log(format!("Uploaded index file to s3 backend: {}", data.name)) + } + self.log_upload_stat( &data.name, &csum, @@ -553,6 +562,16 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload fixed index to s3 backend")?; + self.log(format!( + "Uploaded fixed index file to object store: {}", + data.name + )) + } + self.log_upload_stat( &data.name, &expected_csum, @@ -753,6 +772,25 @@ impl BackupEnvironment { Ok(()) } + + fn s3_upload_index(&self, s3_client: &S3Client, name: &str) -> Result<(), Error> { + let object_key = + pbs_datastore::s3::object_key_from_path(&self.backup_dir.relative_path(), name) + .context("invalid index file object key")?; + + let mut full_path = self.backup_dir.full_path(); + full_path.push(name); + let mut file = std::fs::File::open(&full_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + let data = hyper::body::Bytes::from(buffer); + proxmox_async::runtime::block_on(s3_client.upload_with_retry( + object_key.into(), + data, + true, + ))?; + Ok(()) + } } impl RpcEnvironment for BackupEnvironment { -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel