From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id A68C61FF16B for ; Tue, 15 Jul 2025 14:53:59 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BE48F3C030; Tue, 15 Jul 2025 14:54:24 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 15 Jul 2025 14:52:57 +0200 Message-ID: <20250715125332.954494-20-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250715125332.954494-1-c.ebner@proxmox.com> References: <20250715125332.954494-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.046 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_MSPIKE_H2 0.001 Average reputation (+2) SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v8 10/45] api: backup: conditionally upload indices to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" If the datastore is backed by an S3 compatible object store, upload the dynamic or fixed index files to the object store after closing them. The local index files are kept in the local caching datastore to allow for fast and efficient content lookups, avoiding expensive (as in monetary cost and IO latency) requests. Signed-off-by: Christian Ebner --- changes since version 7: - fix clippy warning and formatting src/api2/backup/environment.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index 3d4677975..9ad13aeb3 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -2,6 +2,7 @@ use anyhow::{bail, format_err, Context, Error}; use pbs_config::BackupLockGuard; use std::collections::HashMap; +use std::io::Read; use std::sync::{Arc, Mutex}; use tracing::info; @@ -18,6 +19,7 @@ use pbs_datastore::dynamic_index::DynamicIndexWriter; use pbs_datastore::fixed_index::FixedIndexWriter; use pbs_datastore::{DataBlob, DataStore, DatastoreBackend}; use proxmox_rest_server::{formatter::*, WorkerTask}; +use proxmox_s3_client::S3Client; use crate::backup::VerifyWorker; @@ -479,6 +481,13 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload dynamic index to s3 backend")?; + self.log(format!("Uploaded index file to s3 backend: {}", data.name)) + } + self.log_upload_stat( &data.name, &csum, @@ -553,6 +562,16 @@ impl BackupEnvironment { ); } + // For S3 backends, upload the index file to the object store after closing + if let DatastoreBackend::S3(s3_client) = &self.backend { + self.s3_upload_index(s3_client, &data.name) + .context("failed to upload fixed index to s3 backend")?; + self.log(format!( + "Uploaded fixed index file to object store: {}", + data.name + )) + } + self.log_upload_stat( &data.name, &expected_csum, @@ -753,6 +772,21 @@ impl BackupEnvironment { Ok(()) } + + fn s3_upload_index(&self, s3_client: &S3Client, name: &str) -> Result<(), Error> { + let object_key = + pbs_datastore::s3::object_key_from_path(&self.backup_dir.relative_path(), name) + .context("invalid index file object key")?; + + let mut full_path = self.backup_dir.full_path(); + full_path.push(name); + let mut file = std::fs::File::open(&full_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + let data = hyper::body::Bytes::from(buffer); + proxmox_async::runtime::block_on(s3_client.upload_with_retry(object_key, data, true))?; + Ok(()) + } } impl RpcEnvironment for BackupEnvironment { -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel